hns_roce_hw_v2.c

   1/*
   2 * Copyright (c) 2016-2017 Hisilicon Limited.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#include <linux/acpi.h>
  34#include <linux/etherdevice.h>
  35#include <linux/interrupt.h>
  36#include <linux/iopoll.h>
  37#include <linux/kernel.h>
  38#include <linux/types.h>
  39#include <linux/workqueue.h>
  40#include <net/addrconf.h>
  41#include <rdma/ib_addr.h>
  42#include <rdma/ib_cache.h>
  43#include <rdma/ib_umem.h>
  44#include <rdma/uverbs_ioctl.h>
  45
  46#include "hclge_main.h"
  47#include "hns_roce_common.h"
  48#include "hns_roce_device.h"
  49#include "hns_roce_cmd.h"
  50#include "hns_roce_hem.h"
  51#include "hns_roce_hw_v2.h"
  52#include "hns_roce_bond.h"
  53
  54#define CREATE_TRACE_POINTS
  55#include "hns_roce_trace.h"
  56
  57enum {
  58	CMD_RST_PRC_OTHERS,
  59	CMD_RST_PRC_SUCCESS,
  60	CMD_RST_PRC_EBUSY,
  61};
  62
  63enum ecc_resource_type {
  64	ECC_RESOURCE_QPC,
  65	ECC_RESOURCE_CQC,
  66	ECC_RESOURCE_MPT,
  67	ECC_RESOURCE_SRQC,
  68	ECC_RESOURCE_GMV,
  69	ECC_RESOURCE_QPC_TIMER,
  70	ECC_RESOURCE_CQC_TIMER,
  71	ECC_RESOURCE_SCCC,
  72	ECC_RESOURCE_COUNT,
  73};
  74
  75static const struct {
  76	const char *name;
  77	u8 read_bt0_op;
  78	u8 write_bt0_op;
  79} fmea_ram_res[] = {
  80	{ "ECC_RESOURCE_QPC",
  81	  HNS_ROCE_CMD_READ_QPC_BT0, HNS_ROCE_CMD_WRITE_QPC_BT0 },
  82	{ "ECC_RESOURCE_CQC",
  83	  HNS_ROCE_CMD_READ_CQC_BT0, HNS_ROCE_CMD_WRITE_CQC_BT0 },
  84	{ "ECC_RESOURCE_MPT",
  85	  HNS_ROCE_CMD_READ_MPT_BT0, HNS_ROCE_CMD_WRITE_MPT_BT0 },
  86	{ "ECC_RESOURCE_SRQC",
  87	  HNS_ROCE_CMD_READ_SRQC_BT0, HNS_ROCE_CMD_WRITE_SRQC_BT0 },
  88	/* ECC_RESOURCE_GMV is handled by cmdq, not mailbox */
  89	{ "ECC_RESOURCE_GMV",
  90	  0, 0 },
  91	{ "ECC_RESOURCE_QPC_TIMER",
  92	  HNS_ROCE_CMD_READ_QPC_TIMER_BT0, HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0 },
  93	{ "ECC_RESOURCE_CQC_TIMER",
  94	  HNS_ROCE_CMD_READ_CQC_TIMER_BT0, HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0 },
  95	{ "ECC_RESOURCE_SCCC",
  96	  HNS_ROCE_CMD_READ_SCCC_BT0, HNS_ROCE_CMD_WRITE_SCCC_BT0 },
  97};
  98
  99static inline void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
 100				   struct ib_sge *sg)
 101{
 102	dseg->lkey = cpu_to_le32(sg->lkey);
 103	dseg->addr = cpu_to_le64(sg->addr);
 104	dseg->len  = cpu_to_le32(sg->length);
 105}
 106
 107/*
 108 * mapped-value = 1 + real-value
 109 * The hns wr opcode real value is start from 0, In order to distinguish between
 110 * initialized and uninitialized map values, we plus 1 to the actual value when
 111 * defining the mapping, so that the validity can be identified by checking the
 112 * mapped value is greater than 0.
 113 */
 114#define HR_OPC_MAP(ib_key, hr_key) \
 115		[IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key
 116
 117static const u32 hns_roce_op_code[] = {
 118	HR_OPC_MAP(RDMA_WRITE,			RDMA_WRITE),
 119	HR_OPC_MAP(RDMA_WRITE_WITH_IMM,		RDMA_WRITE_WITH_IMM),
 120	HR_OPC_MAP(SEND,			SEND),
 121	HR_OPC_MAP(SEND_WITH_IMM,		SEND_WITH_IMM),
 122	HR_OPC_MAP(RDMA_READ,			RDMA_READ),
 123	HR_OPC_MAP(ATOMIC_CMP_AND_SWP,		ATOM_CMP_AND_SWAP),
 124	HR_OPC_MAP(ATOMIC_FETCH_AND_ADD,	ATOM_FETCH_AND_ADD),
 125	HR_OPC_MAP(SEND_WITH_INV,		SEND_WITH_INV),
 126	HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP,	ATOM_MSK_CMP_AND_SWAP),
 127	HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD,	ATOM_MSK_FETCH_AND_ADD),
 128	HR_OPC_MAP(REG_MR,			FAST_REG_PMR),
 129};
 130
 131static u32 to_hr_opcode(u32 ib_opcode)
 132{
 133	if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code))
 134		return HNS_ROCE_V2_WQE_OP_MASK;
 135
 136	return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 :
 137					     HNS_ROCE_V2_WQE_OP_MASK;
 138}
 139
 140static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 141			 const struct ib_reg_wr *wr)
 142{
 143	struct hns_roce_wqe_frmr_seg *fseg =
 144		(void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
 145	struct hns_roce_mr *mr = to_hr_mr(wr->mr);
 146	u64 pbl_ba;
 147
 148	/* use ib_access_flags */
 149	hr_reg_write_bool(fseg, FRMR_BIND_EN, 0);
 150	hr_reg_write_bool(fseg, FRMR_ATOMIC,
 151			  wr->access & IB_ACCESS_REMOTE_ATOMIC);
 152	hr_reg_write_bool(fseg, FRMR_RR, wr->access & IB_ACCESS_REMOTE_READ);
 153	hr_reg_write_bool(fseg, FRMR_RW, wr->access & IB_ACCESS_REMOTE_WRITE);
 154	hr_reg_write_bool(fseg, FRMR_LW, wr->access & IB_ACCESS_LOCAL_WRITE);
 155
 156	/* Data structure reuse may lead to confusion */
 157	pbl_ba = mr->pbl_mtr.hem_cfg.root_ba;
 158	rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba));
 159	rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba));
 160
 161	rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
 162	rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
 163	rc_sq_wqe->rkey = cpu_to_le32(wr->key);
 164	rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
 165
 166	hr_reg_write(fseg, FRMR_PBL_SIZE, mr->npages);
 167	hr_reg_write(fseg, FRMR_PBL_BUF_PG_SZ,
 168		     to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
 169	hr_reg_clear(fseg, FRMR_BLK_MODE);
 170	hr_reg_clear(fseg, FRMR_BLOCK_SIZE);
 171	hr_reg_clear(fseg, FRMR_ZBVA);
 172}
 173
 174static void set_atomic_seg(const struct ib_send_wr *wr,
 175			   struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 176			   unsigned int valid_num_sge)
 177{
 178	struct hns_roce_v2_wqe_data_seg *dseg =
 179		(void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
 180	struct hns_roce_wqe_atomic_seg *aseg =
 181		(void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);
 182
 183	set_data_seg_v2(dseg, wr->sg_list);
 184
 185	if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
 186		aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);
 187		aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add);
 188	} else {
 189		aseg->fetchadd_swap_data =
 190			cpu_to_le64(atomic_wr(wr)->compare_add);
 191		aseg->cmp_data = 0;
 192	}
 193
 194	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge);
 195}
 196
 197static int fill_ext_sge_inl_data(struct hns_roce_qp *qp,
 198				 const struct ib_send_wr *wr,
 199				 unsigned int *sge_idx, u32 msg_len)
 200{
 201	struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev;
 202	unsigned int left_len_in_pg;
 203	unsigned int idx = *sge_idx;
 204	unsigned int i = 0;
 205	unsigned int len;
 206	void *addr;
 207	void *dseg;
 208
 209	if (msg_len > qp->sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) {
 210		ibdev_err(ibdev,
 211			  "no enough extended sge space for inline data.\n");
 212		return -EINVAL;
 213	}
 214
 215	dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
 216	left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg;
 217	len = wr->sg_list[0].length;
 218	addr = (void *)(unsigned long)(wr->sg_list[0].addr);
 219
 220	/* When copying data to extended sge space, the left length in page may
 221	 * not long enough for current user's sge. So the data should be
 222	 * splited into several parts, one in the first page, and the others in
 223	 * the subsequent pages.
 224	 */
 225	while (1) {
 226		if (len <= left_len_in_pg) {
 227			memcpy(dseg, addr, len);
 228
 229			idx += len / HNS_ROCE_SGE_SIZE;
 230
 231			i++;
 232			if (i >= wr->num_sge)
 233				break;
 234
 235			left_len_in_pg -= len;
 236			len = wr->sg_list[i].length;
 237			addr = (void *)(unsigned long)(wr->sg_list[i].addr);
 238			dseg += len;
 239		} else {
 240			memcpy(dseg, addr, left_len_in_pg);
 241
 242			len -= left_len_in_pg;
 243			addr += left_len_in_pg;
 244			idx += left_len_in_pg / HNS_ROCE_SGE_SIZE;
 245			dseg = hns_roce_get_extend_sge(qp,
 246						idx & (qp->sge.sge_cnt - 1));
 247			left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT;
 248		}
 249	}
 250
 251	*sge_idx = idx;
 252
 253	return 0;
 254}
 255
 256static void set_extend_sge(struct hns_roce_qp *qp, struct ib_sge *sge,
 257			   unsigned int *sge_ind, unsigned int cnt)
 258{
 259	struct hns_roce_v2_wqe_data_seg *dseg;
 260	unsigned int idx = *sge_ind;
 261
 262	while (cnt > 0) {
 263		dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
 264		if (likely(sge->length)) {
 265			set_data_seg_v2(dseg, sge);
 266			idx++;
 267			cnt--;
 268		}
 269		sge++;
 270	}
 271
 272	*sge_ind = idx;
 273}
 274
 275static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len)
 276{
 277	struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 278	int mtu = ib_mtu_enum_to_int(qp->path_mtu);
 279
 280	if (mtu < 0 || len > qp->max_inline_data || len > mtu) {
 281		ibdev_err(&hr_dev->ib_dev,
 282			  "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n",
 283			  len, qp->max_inline_data, mtu);
 284		return false;
 285	}
 286
 287	return true;
 288}
 289
 290static int set_rc_inl(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
 291		      struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 292		      unsigned int *sge_idx)
 293{
 294	struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 295	u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len);
 296	struct ib_device *ibdev = &hr_dev->ib_dev;
 297	unsigned int curr_idx = *sge_idx;
 298	void *dseg = rc_sq_wqe;
 299	unsigned int i;
 300	int ret;
 301
 302	if (unlikely(wr->opcode == IB_WR_RDMA_READ)) {
 303		ibdev_err(ibdev, "invalid inline parameters!\n");
 304		return -EINVAL;
 305	}
 306
 307	if (!check_inl_data_len(qp, msg_len))
 308		return -EINVAL;
 309
 310	dseg += sizeof(struct hns_roce_v2_rc_send_wqe);
 311
 312	if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) {
 313		hr_reg_clear(rc_sq_wqe, RC_SEND_WQE_INL_TYPE);
 314
 315		for (i = 0; i < wr->num_sge; i++) {
 316			memcpy(dseg, ((void *)wr->sg_list[i].addr),
 317			       wr->sg_list[i].length);
 318			dseg += wr->sg_list[i].length;
 319		}
 320	} else {
 321		hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_INL_TYPE);
 322
 323		ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len);
 324		if (ret)
 325			return ret;
 326
 327		hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, curr_idx - *sge_idx);
 328	}
 329
 330	*sge_idx = curr_idx;
 331
 332	return 0;
 333}
 334
 335static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 336			     struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 337			     unsigned int *sge_ind,
 338			     unsigned int valid_num_sge)
 339{
 340	struct hns_roce_v2_wqe_data_seg *dseg =
 341		(void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
 342	struct hns_roce_qp *qp = to_hr_qp(ibqp);
 343	int j = 0;
 344	int i;
 345
 346	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_INLINE,
 347		     !!(wr->send_flags & IB_SEND_INLINE));
 348	if (wr->send_flags & IB_SEND_INLINE)
 349		return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind);
 350
 351	if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) {
 352		for (i = 0; i < wr->num_sge; i++) {
 353			if (likely(wr->sg_list[i].length)) {
 354				set_data_seg_v2(dseg, wr->sg_list + i);
 355				dseg++;
 356			}
 357		}
 358	} else {
 359		for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) {
 360			if (likely(wr->sg_list[i].length)) {
 361				set_data_seg_v2(dseg, wr->sg_list + i);
 362				dseg++;
 363				j++;
 364			}
 365		}
 366
 367		set_extend_sge(qp, wr->sg_list + i, sge_ind,
 368			       valid_num_sge - HNS_ROCE_SGE_IN_WQE);
 369	}
 370
 371	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge);
 372
 373	return 0;
 374}
 375
 376static int check_send_valid(struct hns_roce_dev *hr_dev,
 377			    struct hns_roce_qp *hr_qp)
 378{
 379	if (unlikely(hr_qp->state == IB_QPS_RESET ||
 380		     hr_qp->state == IB_QPS_INIT ||
 381		     hr_qp->state == IB_QPS_RTR))
 382		return -EINVAL;
 383	else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))
 384		return -EIO;
 385
 386	return 0;
 387}
 388
 389static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr,
 390				    unsigned int *sge_len)
 391{
 392	unsigned int valid_num = 0;
 393	unsigned int len = 0;
 394	int i;
 395
 396	for (i = 0; i < wr->num_sge; i++) {
 397		if (likely(wr->sg_list[i].length)) {
 398			len += wr->sg_list[i].length;
 399			valid_num++;
 400		}
 401	}
 402
 403	*sge_len = len;
 404	return valid_num;
 405}
 406
 407static __le32 get_immtdata(const struct ib_send_wr *wr)
 408{
 409	switch (wr->opcode) {
 410	case IB_WR_SEND_WITH_IMM:
 411	case IB_WR_RDMA_WRITE_WITH_IMM:
 412		return cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
 413	default:
 414		return 0;
 415	}
 416}
 417
 418static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 419			 const struct ib_send_wr *wr)
 420{
 421	u32 ib_op = wr->opcode;
 422
 423	if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM)
 424		return -EINVAL;
 425
 426	ud_sq_wqe->immtdata = get_immtdata(wr);
 427
 428	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OPCODE, to_hr_opcode(ib_op));
 429
 430	return 0;
 431}
 432
 433static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 434		      struct hns_roce_ah *ah)
 435{
 436	struct ib_device *ib_dev = ah->ibah.device;
 437	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
 438
 439	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_UDPSPN, ah->av.udp_sport);
 440	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit);
 441	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass);
 442	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel);
 443	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl);
 444
 445	ud_sq_wqe->sgid_index = ah->av.gid_index;
 446
 447	memcpy(ud_sq_wqe->dmac, ah->av.mac, ETH_ALEN);
 448	memcpy(ud_sq_wqe->dgid, ah->av.dgid, GID_LEN_V2);
 449
 450	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
 451		return 0;
 452
 453	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN_EN, ah->av.vlan_en);
 454	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN, ah->av.vlan_id);
 455
 456	return 0;
 457}
 458
 459static inline int set_ud_wqe(struct hns_roce_qp *qp,
 460			     const struct ib_send_wr *wr,
 461			     void *wqe, unsigned int *sge_idx,
 462			     unsigned int owner_bit)
 463{
 464	struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
 465	struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;
 466	unsigned int curr_idx = *sge_idx;
 467	unsigned int valid_num_sge;
 468	u32 msg_len = 0;
 469	int ret;
 470
 471	valid_num_sge = calc_wr_sge_num(wr, &msg_len);
 472
 473	ret = set_ud_opcode(ud_sq_wqe, wr);
 474	if (WARN_ON_ONCE(ret))
 475		return ret;
 476
 477	ud_sq_wqe->msg_len = cpu_to_le32(msg_len);
 478
 479	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_CQE,
 480		     !!(wr->send_flags & IB_SEND_SIGNALED));
 481	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SE,
 482		     !!(wr->send_flags & IB_SEND_SOLICITED));
 483
 484	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_PD, to_hr_pd(qp->ibqp.pd)->pdn);
 485	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SGE_NUM, valid_num_sge);
 486	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_MSG_START_SGE_IDX,
 487		     curr_idx & (qp->sge.sge_cnt - 1));
 488
 489	ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
 490			  qp->qkey : ud_wr(wr)->remote_qkey);
 491	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)->remote_qpn);
 492
 493	ret = fill_ud_av(ud_sq_wqe, ah);
 494	if (ret)
 495		return ret;
 496
 497	qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl;
 498
 499	set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
 500
 501	/*
 502	 * The pipeline can sequentially post all valid WQEs into WQ buffer,
 503	 * including new WQEs waiting for the doorbell to update the PI again.
 504	 * Therefore, the owner bit of WQE MUST be updated after all fields
 505	 * and extSGEs have been written into DDR instead of cache.
 506	 */
 507	if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
 508		dma_wmb();
 509
 510	*sge_idx = curr_idx;
 511	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OWNER, owner_bit);
 512
 513	return 0;
 514}
 515
 516static int set_rc_opcode(struct hns_roce_dev *hr_dev,
 517			 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 518			 const struct ib_send_wr *wr)
 519{
 520	u32 ib_op = wr->opcode;
 521	int ret = 0;
 522
 523	rc_sq_wqe->immtdata = get_immtdata(wr);
 524
 525	switch (ib_op) {
 526	case IB_WR_RDMA_READ:
 527	case IB_WR_RDMA_WRITE:
 528	case IB_WR_RDMA_WRITE_WITH_IMM:
 529		rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);
 530		rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);
 531		break;
 532	case IB_WR_SEND:
 533	case IB_WR_SEND_WITH_IMM:
 534		break;
 535	case IB_WR_ATOMIC_CMP_AND_SWP:
 536	case IB_WR_ATOMIC_FETCH_AND_ADD:
 537		rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey);
 538		rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr);
 539		break;
 540	case IB_WR_REG_MR:
 541		if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
 542			set_frmr_seg(rc_sq_wqe, reg_wr(wr));
 543		else
 544			ret = -EOPNOTSUPP;
 545		break;
 546	case IB_WR_SEND_WITH_INV:
 547		rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);
 548		break;
 549	default:
 550		ret = -EINVAL;
 551	}
 552
 553	if (unlikely(ret))
 554		return ret;
 555
 556	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OPCODE, to_hr_opcode(ib_op));
 557
 558	return ret;
 559}
 560
 561static inline int set_rc_wqe(struct hns_roce_qp *qp,
 562			     const struct ib_send_wr *wr,
 563			     void *wqe, unsigned int *sge_idx,
 564			     unsigned int owner_bit)
 565{
 566	struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 567	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
 568	unsigned int curr_idx = *sge_idx;
 569	unsigned int valid_num_sge;
 570	u32 msg_len = 0;
 571	int ret;
 572
 573	valid_num_sge = calc_wr_sge_num(wr, &msg_len);
 574
 575	rc_sq_wqe->msg_len = cpu_to_le32(msg_len);
 576
 577	ret = set_rc_opcode(hr_dev, rc_sq_wqe, wr);
 578	if (WARN_ON_ONCE(ret))
 579		return ret;
 580
 581	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SO,
 582		     (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
 583
 584	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SE,
 585		     (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
 586
 587	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_CQE,
 588		     (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
 589
 590	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_MSG_START_SGE_IDX,
 591		     curr_idx & (qp->sge.sge_cnt - 1));
 592
 593	if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
 594	    wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
 595		if (msg_len != ATOMIC_WR_LEN)
 596			return -EINVAL;
 597		set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);
 598	} else if (wr->opcode != IB_WR_REG_MR) {
 599		ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,
 600					&curr_idx, valid_num_sge);
 601		if (ret)
 602			return ret;
 603	}
 604
 605	/*
 606	 * The pipeline can sequentially post all valid WQEs into WQ buffer,
 607	 * including new WQEs waiting for the doorbell to update the PI again.
 608	 * Therefore, the owner bit of WQE MUST be updated after all fields
 609	 * and extSGEs have been written into DDR instead of cache.
 610	 */
 611	if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
 612		dma_wmb();
 613
 614	*sge_idx = curr_idx;
 615	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OWNER, owner_bit);
 616
 617	return ret;
 618}
 619
 620static inline void update_sq_db(struct hns_roce_dev *hr_dev,
 621				struct hns_roce_qp *qp)
 622{
 623	if (unlikely(qp->state == IB_QPS_ERR)) {
 624		flush_cqe(hr_dev, qp);
 625	} else {
 626		struct hns_roce_v2_db sq_db = {};
 627
 628		hr_reg_write(&sq_db, DB_TAG, qp->qpn);
 629		hr_reg_write(&sq_db, DB_CMD, HNS_ROCE_V2_SQ_DB);
 630		hr_reg_write(&sq_db, DB_PI, qp->sq.head);
 631		hr_reg_write(&sq_db, DB_SL, qp->sl);
 632
 633		hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg);
 634	}
 635}
 636
 637static inline void update_rq_db(struct hns_roce_dev *hr_dev,
 638				struct hns_roce_qp *qp)
 639{
 640	if (unlikely(qp->state == IB_QPS_ERR)) {
 641		flush_cqe(hr_dev, qp);
 642	} else {
 643		if (likely(qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)) {
 644			*qp->rdb.db_record =
 645					qp->rq.head & V2_DB_PRODUCER_IDX_M;
 646		} else {
 647			struct hns_roce_v2_db rq_db = {};
 648
 649			hr_reg_write(&rq_db, DB_TAG, qp->qpn);
 650			hr_reg_write(&rq_db, DB_CMD, HNS_ROCE_V2_RQ_DB);
 651			hr_reg_write(&rq_db, DB_PI, qp->rq.head);
 652
 653			hns_roce_write64(hr_dev, (__le32 *)&rq_db,
 654					 qp->rq.db_reg);
 655		}
 656	}
 657}
 658
 659static void hns_roce_write512(struct hns_roce_dev *hr_dev, u64 *val,
 660			      u64 __iomem *dest)
 661{
 662#define HNS_ROCE_WRITE_TIMES 8
 663	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 664	struct hnae3_handle *handle = priv->handle;
 665	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 666	int i;
 667
 668	if (!hr_dev->dis_db && !ops->get_hw_reset_stat(handle))
 669		for (i = 0; i < HNS_ROCE_WRITE_TIMES; i++)
 670			writeq_relaxed(*(val + i), dest + i);
 671}
 672
 673static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,
 674		       void *wqe)
 675{
 676#define HNS_ROCE_SL_SHIFT 2
 677	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
 678
 679	if (unlikely(qp->state == IB_QPS_ERR)) {
 680		flush_cqe(hr_dev, qp);
 681		return;
 682	}
 683	/* All kinds of DirectWQE have the same header field layout */
 684	hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_FLAG);
 685	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_L, qp->sl);
 686	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_H,
 687		     qp->sl >> HNS_ROCE_SL_SHIFT);
 688	hr_reg_write(rc_sq_wqe, RC_SEND_WQE_WQE_INDEX, qp->sq.head);
 689
 690	hns_roce_write512(hr_dev, wqe, qp->sq.db_reg);
 691}
 692
 693static int hns_roce_v2_post_send(struct ib_qp *ibqp,
 694				 const struct ib_send_wr *wr,
 695				 const struct ib_send_wr **bad_wr)
 696{
 697	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 698	struct ib_device *ibdev = &hr_dev->ib_dev;
 699	struct hns_roce_qp *qp = to_hr_qp(ibqp);
 700	unsigned long flags = 0;
 701	unsigned int owner_bit;
 702	unsigned int sge_idx;
 703	unsigned int wqe_idx;
 704	void *wqe = NULL;
 705	u32 nreq;
 706	int ret;
 707
 708	spin_lock_irqsave(&qp->sq.lock, flags);
 709
 710	ret = check_send_valid(hr_dev, qp);
 711	if (unlikely(ret)) {
 712		*bad_wr = wr;
 713		nreq = 0;
 714		goto out;
 715	}
 716
 717	sge_idx = qp->next_sge;
 718
 719	for (nreq = 0; wr; ++nreq, wr = wr->next) {
 720		if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
 721			ret = -ENOMEM;
 722			*bad_wr = wr;
 723			goto out;
 724		}
 725
 726		wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);
 727
 728		if (unlikely(wr->num_sge > qp->sq.max_gs)) {
 729			ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n",
 730				  wr->num_sge, qp->sq.max_gs);
 731			ret = -EINVAL;
 732			*bad_wr = wr;
 733			goto out;
 734		}
 735
 736		wqe = hns_roce_get_send_wqe(qp, wqe_idx);
 737		qp->sq.wrid[wqe_idx] = wr->wr_id;
 738		owner_bit =
 739		       ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
 740
 741		/* RC and UD share the same DirectWQE field layout */
 742		((struct hns_roce_v2_rc_send_wqe *)wqe)->byte_4 = 0;
 743
 744		/* Corresponding to the QP type, wqe process separately */
 745		if (ibqp->qp_type == IB_QPT_RC)
 746			ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);
 747		else
 748			ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit);
 749
 750		trace_hns_sq_wqe(qp->qpn, wqe_idx, wqe, 1 << qp->sq.wqe_shift,
 751				 wr->wr_id, TRACE_SQ);
 752		if (unlikely(ret)) {
 753			*bad_wr = wr;
 754			goto out;
 755		}
 756	}
 757
 758out:
 759	if (likely(nreq)) {
 760		qp->sq.head += nreq;
 761		qp->next_sge = sge_idx;
 762
 763		if (nreq == 1 && !ret &&
 764		    (qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
 765			write_dwqe(hr_dev, qp, wqe);
 766		else
 767			update_sq_db(hr_dev, qp);
 768	}
 769
 770	spin_unlock_irqrestore(&qp->sq.lock, flags);
 771
 772	return ret;
 773}
 774
 775static int check_recv_valid(struct hns_roce_dev *hr_dev,
 776			    struct hns_roce_qp *hr_qp)
 777{
 778	if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))
 779		return -EIO;
 780
 781	if (hr_qp->state == IB_QPS_RESET)
 782		return -EINVAL;
 783
 784	return 0;
 785}
 786
 787static void fill_recv_sge_to_wqe(const struct ib_recv_wr *wr, void *wqe,
 788				 u32 max_sge, bool rsv)
 789{
 790	struct hns_roce_v2_wqe_data_seg *dseg = wqe;
 791	u32 i, cnt;
 792
 793	for (i = 0, cnt = 0; i < wr->num_sge; i++) {
 794		/* Skip zero-length sge */
 795		if (!wr->sg_list[i].length)
 796			continue;
 797		set_data_seg_v2(dseg + cnt, wr->sg_list + i);
 798		cnt++;
 799	}
 800
 801	/* Fill a reserved sge to make hw stop reading remaining segments */
 802	if (rsv) {
 803		dseg[cnt].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
 804		dseg[cnt].addr = 0;
 805		dseg[cnt].len = cpu_to_le32(HNS_ROCE_INVALID_SGE_LENGTH);
 806	} else {
 807		/* Clear remaining segments to make ROCEE ignore sges */
 808		if (cnt < max_sge)
 809			memset(dseg + cnt, 0,
 810			       (max_sge - cnt) * HNS_ROCE_SGE_SIZE);
 811	}
 812}
 813
 814static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr,
 815			u32 wqe_idx, u32 max_sge)
 816{
 817	void *wqe = NULL;
 818
 819	wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
 820	fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge);
 821
 822	trace_hns_rq_wqe(hr_qp->qpn, wqe_idx, wqe, 1 << hr_qp->rq.wqe_shift,
 823			 wr->wr_id, TRACE_RQ);
 824}
 825
 826static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
 827				 const struct ib_recv_wr *wr,
 828				 const struct ib_recv_wr **bad_wr)
 829{
 830	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 831	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 832	struct ib_device *ibdev = &hr_dev->ib_dev;
 833	u32 wqe_idx, nreq, max_sge;
 834	unsigned long flags;
 835	int ret;
 836
 837	spin_lock_irqsave(&hr_qp->rq.lock, flags);
 838
 839	ret = check_recv_valid(hr_dev, hr_qp);
 840	if (unlikely(ret)) {
 841		*bad_wr = wr;
 842		nreq = 0;
 843		goto out;
 844	}
 845
 846	max_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
 847	for (nreq = 0; wr; ++nreq, wr = wr->next) {
 848		if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq,
 849						  hr_qp->ibqp.recv_cq))) {
 850			ret = -ENOMEM;
 851			*bad_wr = wr;
 852			goto out;
 853		}
 854
 855		if (unlikely(wr->num_sge > max_sge)) {
 856			ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n",
 857				  wr->num_sge, max_sge);
 858			ret = -EINVAL;
 859			*bad_wr = wr;
 860			goto out;
 861		}
 862
 863		wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);
 864		fill_rq_wqe(hr_qp, wr, wqe_idx, max_sge);
 865		hr_qp->rq.wrid[wqe_idx] = wr->wr_id;
 866	}
 867
 868out:
 869	if (likely(nreq)) {
 870		hr_qp->rq.head += nreq;
 871
 872		update_rq_db(hr_dev, hr_qp);
 873	}
 874	spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 875
 876	return ret;
 877}
 878
 879static void *get_srq_wqe_buf(struct hns_roce_srq *srq, u32 n)
 880{
 881	return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift);
 882}
 883
 884static void *get_idx_buf(struct hns_roce_idx_que *idx_que, u32 n)
 885{
 886	return hns_roce_buf_offset(idx_que->mtr.kmem,
 887				   n << idx_que->entry_shift);
 888}
 889
 890static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, u32 wqe_index)
 891{
 892	/* always called with interrupts disabled. */
 893	spin_lock(&srq->lock);
 894
 895	bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
 896	srq->idx_que.tail++;
 897
 898	spin_unlock(&srq->lock);
 899}
 900
 901static int hns_roce_srqwq_overflow(struct hns_roce_srq *srq)
 902{
 903	struct hns_roce_idx_que *idx_que = &srq->idx_que;
 904
 905	return idx_que->head - idx_que->tail >= srq->wqe_cnt;
 906}
 907
 908static int check_post_srq_valid(struct hns_roce_srq *srq, u32 max_sge,
 909				const struct ib_recv_wr *wr)
 910{
 911	struct ib_device *ib_dev = srq->ibsrq.device;
 912
 913	if (unlikely(wr->num_sge > max_sge)) {
 914		ibdev_err(ib_dev,
 915			  "failed to check sge, wr->num_sge = %d, max_sge = %u.\n",
 916			  wr->num_sge, max_sge);
 917		return -EINVAL;
 918	}
 919
 920	if (unlikely(hns_roce_srqwq_overflow(srq))) {
 921		ibdev_err(ib_dev,
 922			  "failed to check srqwq status, srqwq is full.\n");
 923		return -ENOMEM;
 924	}
 925
 926	return 0;
 927}
 928
 929static int get_srq_wqe_idx(struct hns_roce_srq *srq, u32 *wqe_idx)
 930{
 931	struct hns_roce_idx_que *idx_que = &srq->idx_que;
 932	u32 pos;
 933
 934	pos = find_first_zero_bit(idx_que->bitmap, srq->wqe_cnt);
 935	if (unlikely(pos == srq->wqe_cnt))
 936		return -ENOSPC;
 937
 938	bitmap_set(idx_que->bitmap, pos, 1);
 939	*wqe_idx = pos;
 940	return 0;
 941}
 942
 943static void fill_wqe_idx(struct hns_roce_srq *srq, unsigned int wqe_idx)
 944{
 945	struct hns_roce_idx_que *idx_que = &srq->idx_que;
 946	unsigned int head;
 947	__le32 *buf;
 948
 949	head = idx_que->head & (srq->wqe_cnt - 1);
 950
 951	buf = get_idx_buf(idx_que, head);
 952	*buf = cpu_to_le32(wqe_idx);
 953
 954	idx_que->head++;
 955}
 956
 957static void update_srq_db(struct hns_roce_srq *srq)
 958{
 959	struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);
 960	struct hns_roce_v2_db db = {};
 961
 962	hr_reg_write(&db, DB_TAG, srq->srqn);
 963	hr_reg_write(&db, DB_CMD, HNS_ROCE_V2_SRQ_DB);
 964	hr_reg_write(&db, DB_PI, srq->idx_que.head);
 965
 966	hns_roce_write64(hr_dev, (__le32 *)&db, srq->db_reg);
 967}
 968
 969static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
 970				     const struct ib_recv_wr *wr,
 971				     const struct ib_recv_wr **bad_wr)
 972{
 973	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
 974	unsigned long flags;
 975	int ret = 0;
 976	u32 max_sge;
 977	u32 wqe_idx;
 978	void *wqe;
 979	u32 nreq;
 980
 981	spin_lock_irqsave(&srq->lock, flags);
 982
 983	max_sge = srq->max_gs - srq->rsv_sge;
 984	for (nreq = 0; wr; ++nreq, wr = wr->next) {
 985		ret = check_post_srq_valid(srq, max_sge, wr);
 986		if (ret) {
 987			*bad_wr = wr;
 988			break;
 989		}
 990
 991		ret = get_srq_wqe_idx(srq, &wqe_idx);
 992		if (unlikely(ret)) {
 993			*bad_wr = wr;
 994			break;
 995		}
 996
 997		wqe = get_srq_wqe_buf(srq, wqe_idx);
 998		fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge);
 999		fill_wqe_idx(srq, wqe_idx);
1000		srq->wrid[wqe_idx] = wr->wr_id;
1001
1002		trace_hns_srq_wqe(srq->srqn, wqe_idx, wqe, 1 << srq->wqe_shift,
1003				  wr->wr_id, TRACE_SRQ);
1004	}
1005
1006	if (likely(nreq)) {
1007		if (srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB)
1008			*srq->rdb.db_record = srq->idx_que.head &
1009					      V2_DB_PRODUCER_IDX_M;
1010		else
1011			update_srq_db(srq);
1012	}
1013
1014	spin_unlock_irqrestore(&srq->lock, flags);
1015
1016	return ret;
1017}
1018
1019static u32 hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
1020				      unsigned long instance_stage,
1021				      unsigned long reset_stage)
1022{
1023	/* When hardware reset has been completed once or more, we should stop
1024	 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
1025	 * function, we should exit with error. If now at HNAE3_INIT_CLIENT
1026	 * stage of soft reset process, we should exit with error, and then
1027	 * HNAE3_INIT_CLIENT related process can rollback the operation like
1028	 * notifing hardware to free resources, HNAE3_INIT_CLIENT related
1029	 * process will exit with error to notify NIC driver to reschedule soft
1030	 * reset process once again.
1031	 */
1032	hr_dev->is_reset = true;
1033	hr_dev->dis_db = true;
1034
1035	if (reset_stage == HNS_ROCE_STATE_RST_INIT ||
1036	    instance_stage == HNS_ROCE_STATE_INIT)
1037		return CMD_RST_PRC_EBUSY;
1038
1039	return CMD_RST_PRC_SUCCESS;
1040}
1041
1042static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
1043					unsigned long instance_stage,
1044					unsigned long reset_stage)
1045{
1046#define HW_RESET_TIMEOUT_US 1000000
1047#define HW_RESET_SLEEP_US 1000
1048
1049	struct hns_roce_v2_priv *priv = hr_dev->priv;
1050	struct hnae3_handle *handle = priv->handle;
1051	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1052	unsigned long val;
1053	int ret;
1054
1055	/* When hardware reset is detected, we should stop sending mailbox&cmq&
1056	 * doorbell to hardware. If now in .init_instance() function, we should
1057	 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
1058	 * process, we should exit with error, and then HNAE3_INIT_CLIENT
1059	 * related process can rollback the operation like notifing hardware to
1060	 * free resources, HNAE3_INIT_CLIENT related process will exit with
1061	 * error to notify NIC driver to reschedule soft reset process once
1062	 * again.
1063	 */
1064	hr_dev->dis_db = true;
1065
1066	ret = read_poll_timeout(ops->ae_dev_reset_cnt, val,
1067				val > hr_dev->reset_cnt, HW_RESET_SLEEP_US,
1068				HW_RESET_TIMEOUT_US, false, handle);
1069	if (!ret)
1070		hr_dev->is_reset = true;
1071
1072	if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
1073	    instance_stage == HNS_ROCE_STATE_INIT)
1074		return CMD_RST_PRC_EBUSY;
1075
1076	return CMD_RST_PRC_SUCCESS;
1077}
1078
1079static u32 hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
1080{
1081	struct hns_roce_v2_priv *priv = hr_dev->priv;
1082	struct hnae3_handle *handle = priv->handle;
1083	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1084
1085	/* When software reset is detected at .init_instance() function, we
1086	 * should stop sending mailbox&cmq&doorbell to hardware, and exit
1087	 * with error.
1088	 */
1089	hr_dev->dis_db = true;
1090	if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
1091		hr_dev->is_reset = true;
1092
1093	return CMD_RST_PRC_EBUSY;
1094}
1095
1096static u32 check_aedev_reset_status(struct hns_roce_dev *hr_dev,
1097				    struct hnae3_handle *handle)
1098{
1099	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1100	unsigned long instance_stage; /* the current instance stage */
1101	unsigned long reset_stage; /* the current reset stage */
1102	unsigned long reset_cnt;
1103	bool sw_resetting;
1104	bool hw_resetting;
1105
1106	/* Get information about reset from NIC driver or RoCE driver itself,
1107	 * the meaning of the following variables from NIC driver are described
1108	 * as below:
1109	 * reset_cnt -- The count value of completed hardware reset.
1110	 * hw_resetting -- Whether hardware device is resetting now.
1111	 * sw_resetting -- Whether NIC's software reset process is running now.
1112	 */
1113	instance_stage = handle->rinfo.instance_state;
1114	reset_stage = handle->rinfo.reset_state;
1115	reset_cnt = ops->ae_dev_reset_cnt(handle);
1116	if (reset_cnt != hr_dev->reset_cnt)
1117		return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
1118						  reset_stage);
1119
1120	hw_resetting = ops->get_cmdq_stat(handle);
1121	if (hw_resetting)
1122		return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
1123						    reset_stage);
1124
1125	sw_resetting = ops->ae_dev_resetting(handle);
1126	if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
1127		return hns_roce_v2_cmd_sw_resetting(hr_dev);
1128
1129	return CMD_RST_PRC_OTHERS;
1130}
1131
1132static bool check_device_is_in_reset(struct hns_roce_dev *hr_dev)
1133{
1134	struct hns_roce_v2_priv *priv = hr_dev->priv;
1135	struct hnae3_handle *handle = priv->handle;
1136	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1137
1138	if (hr_dev->reset_cnt != ops->ae_dev_reset_cnt(handle))
1139		return true;
1140
1141	if (ops->get_hw_reset_stat(handle))
1142		return true;
1143
1144	if (ops->ae_dev_resetting(handle))
1145		return true;
1146
1147	return false;
1148}
1149
1150static bool v2_chk_mbox_is_avail(struct hns_roce_dev *hr_dev, bool *busy)
1151{
1152	struct hns_roce_v2_priv *priv = hr_dev->priv;
1153	u32 status;
1154
1155	if (hr_dev->is_reset)
1156		status = CMD_RST_PRC_SUCCESS;
1157	else
1158		status = check_aedev_reset_status(hr_dev, priv->handle);
1159
1160	*busy = (status == CMD_RST_PRC_EBUSY);
1161
1162	return status == CMD_RST_PRC_OTHERS;
1163}
1164
1165static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
1166				   struct hns_roce_v2_cmq_ring *ring)
1167{
1168	int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
1169
1170	ring->desc = dma_alloc_coherent(hr_dev->dev, size,
1171					&ring->desc_dma_addr, GFP_KERNEL);
1172	if (!ring->desc)
1173		return -ENOMEM;
1174
1175	return 0;
1176}
1177
1178static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
1179				   struct hns_roce_v2_cmq_ring *ring)
1180{
1181	dma_free_coherent(hr_dev->dev,
1182			  ring->desc_num * sizeof(struct hns_roce_cmq_desc),
1183			  ring->desc, ring->desc_dma_addr);
1184
1185	ring->desc_dma_addr = 0;
1186}
1187
1188static int init_csq(struct hns_roce_dev *hr_dev,
1189		    struct hns_roce_v2_cmq_ring *csq)
1190{
1191	dma_addr_t dma;
1192	int ret;
1193
1194	csq->desc_num = CMD_CSQ_DESC_NUM;
1195	spin_lock_init(&csq->lock);
1196	csq->flag = TYPE_CSQ;
1197	csq->head = 0;
1198
1199	ret = hns_roce_alloc_cmq_desc(hr_dev, csq);
1200	if (ret)
1201		return ret;
1202
1203	dma = csq->desc_dma_addr;
1204	roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, lower_32_bits(dma));
1205	roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, upper_32_bits(dma));
1206	roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
1207		   (u32)csq->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
1208
1209	/* Make sure to write CI first and then PI */
1210	roce_write(hr_dev, ROCEE_TX_CMQ_CI_REG, 0);
1211	roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, 0);
1212
1213	return 0;
1214}
1215
1216static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
1217{
1218	struct hns_roce_v2_priv *priv = hr_dev->priv;
1219	int ret;
1220
1221	priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
1222
1223	ret = init_csq(hr_dev, &priv->cmq.csq);
1224	if (ret)
1225		dev_err(hr_dev->dev, "failed to init CSQ, ret = %d.\n", ret);
1226
1227	return ret;
1228}
1229
1230static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
1231{
1232	struct hns_roce_v2_priv *priv = hr_dev->priv;
1233
1234	hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
1235}
1236
1237static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
1238					  enum hns_roce_opcode_type opcode,
1239					  bool is_read)
1240{
1241	memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
1242	desc->opcode = cpu_to_le16(opcode);
1243	desc->flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN);
1244	if (is_read)
1245		desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
1246	else
1247		desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1248}
1249
1250static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
1251{
1252	u32 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);
1253	struct hns_roce_v2_priv *priv = hr_dev->priv;
1254
1255	return tail == priv->cmq.csq.head;
1256}
1257
1258static void update_cmdq_status(struct hns_roce_dev *hr_dev)
1259{
1260	struct hns_roce_v2_priv *priv = hr_dev->priv;
1261	struct hnae3_handle *handle = priv->handle;
1262
1263	if (handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT ||
1264	    handle->rinfo.instance_state == HNS_ROCE_STATE_INIT)
1265		hr_dev->cmd.state = HNS_ROCE_CMDQ_STATE_FATAL_ERR;
1266}
1267
1268static int hns_roce_cmd_err_convert_errno(u16 desc_ret)
1269{
1270	struct hns_roce_cmd_errcode errcode_table[] = {
1271		{CMD_EXEC_SUCCESS, 0},
1272		{CMD_NO_AUTH, -EPERM},
1273		{CMD_NOT_EXIST, -EOPNOTSUPP},
1274		{CMD_CRQ_FULL, -EXFULL},
1275		{CMD_NEXT_ERR, -ENOSR},
1276		{CMD_NOT_EXEC, -ENOTBLK},
1277		{CMD_PARA_ERR, -EINVAL},
1278		{CMD_RESULT_ERR, -ERANGE},
1279		{CMD_TIMEOUT, -ETIME},
1280		{CMD_HILINK_ERR, -ENOLINK},
1281		{CMD_INFO_ILLEGAL, -ENXIO},
1282		{CMD_INVALID, -EBADR},
1283	};
1284	u16 i;
1285
1286	for (i = 0; i < ARRAY_SIZE(errcode_table); i++)
1287		if (desc_ret == errcode_table[i].return_status)
1288			return errcode_table[i].errno;
1289	return -EIO;
1290}
1291
1292static u32 hns_roce_cmdq_tx_timeout(u16 opcode, u32 tx_timeout)
1293{
1294	static const struct hns_roce_cmdq_tx_timeout_map cmdq_tx_timeout[] = {
1295		{HNS_ROCE_OPC_POST_MB, HNS_ROCE_OPC_POST_MB_TIMEOUT},
1296	};
1297	int i;
1298
1299	for (i = 0; i < ARRAY_SIZE(cmdq_tx_timeout); i++)
1300		if (cmdq_tx_timeout[i].opcode == opcode)
1301			return cmdq_tx_timeout[i].tx_timeout;
1302
1303	return tx_timeout;
1304}
1305
1306static void hns_roce_wait_csq_done(struct hns_roce_dev *hr_dev, u32 tx_timeout)
1307{
1308	u32 timeout = 0;
1309
1310	do {
1311		if (hns_roce_cmq_csq_done(hr_dev))
1312			break;
1313		udelay(1);
1314	} while (++timeout < tx_timeout);
1315}
1316
1317static int __hns_roce_cmq_send_one(struct hns_roce_dev *hr_dev,
1318				   struct hns_roce_cmq_desc *desc,
1319				   int num, u32 tx_timeout)
1320{
1321	struct hns_roce_v2_priv *priv = hr_dev->priv;
1322	struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1323	u16 desc_ret;
1324	u32 tail;
1325	int ret;
1326	int i;
1327
1328	tail = csq->head;
1329
1330	for (i = 0; i < num; i++) {
1331		trace_hns_cmdq_req(hr_dev, &desc[i]);
1332
1333		csq->desc[csq->head++] = desc[i];
1334		if (csq->head == csq->desc_num)
1335			csq->head = 0;
1336	}
1337
1338	/* Write to hardware */
1339	roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, csq->head);
1340
1341	atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CMDS_CNT]);
1342
1343	hns_roce_wait_csq_done(hr_dev, tx_timeout);
1344	if (hns_roce_cmq_csq_done(hr_dev)) {
1345		ret = 0;
1346		for (i = 0; i < num; i++) {
1347			trace_hns_cmdq_resp(hr_dev, &csq->desc[tail]);
1348
1349			/* check the result of hardware write back */
1350			desc_ret = le16_to_cpu(csq->desc[tail++].retval);
1351			if (tail == csq->desc_num)
1352				tail = 0;
1353			if (likely(desc_ret == CMD_EXEC_SUCCESS))
1354				continue;
1355
1356			ret = hns_roce_cmd_err_convert_errno(desc_ret);
1357		}
1358	} else {
1359		/* FW/HW reset or incorrect number of desc */
1360		tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);
1361		dev_warn(hr_dev->dev, "CMDQ move tail from %u to %u.\n",
1362			 csq->head, tail);
1363		csq->head = tail;
1364
1365		update_cmdq_status(hr_dev);
1366
1367		ret = -EAGAIN;
1368	}
1369
1370	if (ret)
1371		atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CMDS_ERR_CNT]);
1372
1373	return ret;
1374}
1375
1376static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1377			       struct hns_roce_cmq_desc *desc, int num)
1378{
1379	struct hns_roce_v2_priv *priv = hr_dev->priv;
1380	struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1381	u16 opcode = le16_to_cpu(desc->opcode);
1382	u32 tx_timeout = hns_roce_cmdq_tx_timeout(opcode, priv->cmq.tx_timeout);
1383	u8 try_cnt = HNS_ROCE_OPC_POST_MB_TRY_CNT;
1384	u32 rsv_tail;
1385	int ret;
1386	int i;
1387
1388	while (try_cnt) {
1389		try_cnt--;
1390
1391		spin_lock_bh(&csq->lock);
1392		rsv_tail = csq->head;
1393		ret = __hns_roce_cmq_send_one(hr_dev, desc, num, tx_timeout);
1394		if (opcode == HNS_ROCE_OPC_POST_MB && ret == -ETIME &&
1395		    try_cnt) {
1396			spin_unlock_bh(&csq->lock);
1397			mdelay(HNS_ROCE_OPC_POST_MB_RETRY_GAP_MSEC);
1398			continue;
1399		}
1400
1401		for (i = 0; i < num; i++) {
1402			desc[i] = csq->desc[rsv_tail++];
1403			if (rsv_tail == csq->desc_num)
1404				rsv_tail = 0;
1405		}
1406		spin_unlock_bh(&csq->lock);
1407		break;
1408	}
1409
1410	if (ret)
1411		dev_err_ratelimited(hr_dev->dev,
1412				    "Cmdq IO error, opcode = 0x%x, return = %d.\n",
1413				    opcode, ret);
1414
1415	return ret;
1416}
1417
1418static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1419			     struct hns_roce_cmq_desc *desc, int num)
1420{
1421	bool busy;
1422	int ret;
1423
1424	if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)
1425		return -EIO;
1426
1427	if (!v2_chk_mbox_is_avail(hr_dev, &busy))
1428		return busy ? -EBUSY : 0;
1429
1430	ret = __hns_roce_cmq_send(hr_dev, desc, num);
1431	if (ret) {
1432		if (!v2_chk_mbox_is_avail(hr_dev, &busy))
1433			return busy ? -EBUSY : 0;
1434	}
1435
1436	return ret;
1437}
1438
1439static enum hns_roce_opcode_type
1440	get_bond_opcode(enum hns_roce_bond_cmd_type bond_type)
1441{
1442	switch (bond_type) {
1443	case HNS_ROCE_SET_BOND:
1444		return HNS_ROCE_OPC_SET_BOND_INFO;
1445	case HNS_ROCE_CHANGE_BOND:
1446		return HNS_ROCE_OPC_CHANGE_ACTIVE_PORT;
1447	case HNS_ROCE_CLEAR_BOND:
1448		return HNS_ROCE_OPC_CLEAR_BOND_INFO;
1449	default:
1450		WARN(true, "Invalid bond type %d!\n", bond_type);
1451		return HNS_ROCE_OPC_SET_BOND_INFO;
1452	}
1453}
1454
1455static enum hns_roce_bond_hashtype
1456	get_bond_hashtype(enum netdev_lag_hash netdev_hashtype)
1457{
1458	switch (netdev_hashtype) {
1459	case NETDEV_LAG_HASH_L2:
1460		return BOND_HASH_L2;
1461	case NETDEV_LAG_HASH_L34:
1462		return BOND_HASH_L34;
1463	case NETDEV_LAG_HASH_L23:
1464		return BOND_HASH_L23;
1465	default:
1466		WARN(true, "Invalid hash type %d!\n", netdev_hashtype);
1467		return BOND_HASH_L2;
1468	}
1469}
1470
1471int hns_roce_cmd_bond(struct hns_roce_bond_group *bond_grp,
1472		      enum hns_roce_bond_cmd_type bond_type)
1473{
1474	enum hns_roce_opcode_type opcode = get_bond_opcode(bond_type);
1475	struct hns_roce_bond_info *slave_info;
1476	struct hns_roce_cmq_desc desc = {};
1477	int ret;
1478
1479	slave_info = (struct hns_roce_bond_info *)desc.data;
1480	hns_roce_cmq_setup_basic_desc(&desc, opcode, false);
1481
1482	slave_info->bond_id = cpu_to_le32(bond_grp->bond_id);
1483	if (bond_type == HNS_ROCE_CLEAR_BOND)
1484		goto out;
1485
1486	if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
1487		slave_info->bond_mode = cpu_to_le32(BOND_MODE_1);
1488		if (bond_grp->active_slave_num != 1)
1489			ibdev_warn(&bond_grp->main_hr_dev->ib_dev,
1490				   "active slave cnt(%u) in Mode 1 is invalid.\n",
1491				   bond_grp->active_slave_num);
1492	} else {
1493		slave_info->bond_mode = cpu_to_le32(BOND_MODE_2_4);
1494		slave_info->hash_policy =
1495			cpu_to_le32(get_bond_hashtype(bond_grp->hash_type));
1496	}
1497
1498	slave_info->active_slave_cnt = cpu_to_le32(bond_grp->active_slave_num);
1499	slave_info->active_slave_mask = cpu_to_le32(bond_grp->active_slave_map);
1500	slave_info->slave_mask = cpu_to_le32(bond_grp->slave_map);
1501
1502out:
1503	ret = hns_roce_cmq_send(bond_grp->main_hr_dev, &desc, 1);
1504	if (ret)
1505		ibdev_err(&bond_grp->main_hr_dev->ib_dev,
1506			  "cmq bond type(%d) failed, ret = %d.\n",
1507			  bond_type, ret);
1508
1509	return ret;
1510}
1511
1512static int config_hem_ba_to_hw(struct hns_roce_dev *hr_dev,
1513			       dma_addr_t base_addr, u8 cmd, unsigned long tag)
1514{
1515	struct hns_roce_cmd_mailbox *mbox;
1516	int ret;
1517
1518	mbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1519	if (IS_ERR(mbox))
1520		return PTR_ERR(mbox);
1521
1522	ret = hns_roce_cmd_mbox(hr_dev, base_addr, mbox->dma, cmd, tag);
1523	hns_roce_free_cmd_mailbox(hr_dev, mbox);
1524	return ret;
1525}
1526
1527static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
1528{
1529	struct hns_roce_query_version *resp;
1530	struct hns_roce_cmq_desc desc;
1531	int ret;
1532
1533	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
1534	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1535	if (ret)
1536		return ret;
1537
1538	resp = (struct hns_roce_query_version *)desc.data;
1539	hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);
1540	hr_dev->vendor_id = hr_dev->pci_dev->vendor;
1541
1542	return 0;
1543}
1544
1545static void func_clr_hw_resetting_state(struct hns_roce_dev *hr_dev,
1546					struct hnae3_handle *handle)
1547{
1548	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1549	unsigned long end;
1550
1551	hr_dev->dis_db = true;
1552
1553	dev_warn(hr_dev->dev,
1554		 "func clear is pending, device in resetting state.\n");
1555	end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1556	while (end) {
1557		if (!ops->get_hw_reset_stat(handle)) {
1558			hr_dev->is_reset = true;
1559			dev_info(hr_dev->dev,
1560				 "func clear success after reset.\n");
1561			return;
1562		}
1563		msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1564		end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1565	}
1566
1567	dev_warn(hr_dev->dev, "func clear failed.\n");
1568}
1569
1570static void func_clr_sw_resetting_state(struct hns_roce_dev *hr_dev,
1571					struct hnae3_handle *handle)
1572{
1573	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1574	unsigned long end;
1575
1576	hr_dev->dis_db = true;
1577
1578	dev_warn(hr_dev->dev,
1579		 "func clear is pending, device in resetting state.\n");
1580	end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1581	while (end) {
1582		if (ops->ae_dev_reset_cnt(handle) !=
1583		    hr_dev->reset_cnt) {
1584			hr_dev->is_reset = true;
1585			dev_info(hr_dev->dev,
1586				 "func clear success after sw reset\n");
1587			return;
1588		}
1589		msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1590		end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1591	}
1592
1593	dev_warn(hr_dev->dev, "func clear failed because of unfinished sw reset\n");
1594}
1595
1596static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval,
1597				       int flag)
1598{
1599	struct hns_roce_v2_priv *priv = hr_dev->priv;
1600	struct hnae3_handle *handle = priv->handle;
1601	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1602
1603	if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) {
1604		hr_dev->dis_db = true;
1605		hr_dev->is_reset = true;
1606		dev_info(hr_dev->dev, "func clear success after reset.\n");
1607		return;
1608	}
1609
1610	if (ops->get_hw_reset_stat(handle)) {
1611		func_clr_hw_resetting_state(hr_dev, handle);
1612		return;
1613	}
1614
1615	if (ops->ae_dev_resetting(handle) &&
1616	    handle->rinfo.instance_state == HNS_ROCE_STATE_INIT) {
1617		func_clr_sw_resetting_state(hr_dev, handle);
1618		return;
1619	}
1620
1621	if (retval && !flag)
1622		dev_warn(hr_dev->dev,
1623			 "func clear read failed, ret = %d.\n", retval);
1624
1625	dev_warn(hr_dev->dev, "func clear failed.\n");
1626}
1627
1628static void __hns_roce_function_clear(struct hns_roce_dev *hr_dev, int vf_id)
1629{
1630	bool fclr_write_fail_flag = false;
1631	struct hns_roce_func_clear *resp;
1632	struct hns_roce_cmq_desc desc;
1633	unsigned long end;
1634	int ret = 0;
1635
1636	if (check_device_is_in_reset(hr_dev))
1637		goto out;
1638
1639	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
1640	resp = (struct hns_roce_func_clear *)desc.data;
1641	resp->rst_funcid_en = cpu_to_le32(vf_id);
1642
1643	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1644	if (ret) {
1645		fclr_write_fail_flag = true;
1646		dev_err(hr_dev->dev, "func clear write failed, ret = %d.\n",
1647			 ret);
1648		goto out;
1649	}
1650
1651	msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
1652	end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
1653	while (end) {
1654		if (check_device_is_in_reset(hr_dev))
1655			goto out;
1656		msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
1657		end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
1658
1659		hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
1660					      true);
1661
1662		resp->rst_funcid_en = cpu_to_le32(vf_id);
1663		ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1664		if (ret)
1665			continue;
1666
1667		if (hr_reg_read(resp, FUNC_CLEAR_RST_FUN_DONE)) {
1668			if (vf_id == 0)
1669				hr_dev->is_reset = true;
1670			return;
1671		}
1672	}
1673
1674out:
1675	hns_roce_func_clr_rst_proc(hr_dev, ret, fclr_write_fail_flag);
1676}
1677
1678static int hns_roce_free_vf_resource(struct hns_roce_dev *hr_dev, int vf_id)
1679{
1680	enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES;
1681	struct hns_roce_cmq_desc desc[2];
1682	struct hns_roce_cmq_req *req_a;
1683
1684	req_a = (struct hns_roce_cmq_req *)desc[0].data;
1685	hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);
1686	desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1687	hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);
1688	hr_reg_write(req_a, FUNC_RES_A_VF_ID, vf_id);
1689
1690	return hns_roce_cmq_send(hr_dev, desc, 2);
1691}
1692
1693static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
1694{
1695	int ret;
1696	int i;
1697
1698	if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)
1699		return;
1700
1701	for (i = hr_dev->func_num - 1; i >= 0; i--) {
1702		__hns_roce_function_clear(hr_dev, i);
1703
1704		if (i == 0)
1705			continue;
1706
1707		ret = hns_roce_free_vf_resource(hr_dev, i);
1708		if (ret)
1709			ibdev_err(&hr_dev->ib_dev,
1710				  "failed to free vf resource, vf_id = %d, ret = %d.\n",
1711				  i, ret);
1712	}
1713}
1714
1715static int hns_roce_clear_extdb_list_info(struct hns_roce_dev *hr_dev)
1716{
1717	struct hns_roce_cmq_desc desc;
1718	int ret;
1719
1720	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLEAR_EXTDB_LIST_INFO,
1721				      false);
1722	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1723	if (ret)
1724		ibdev_err(&hr_dev->ib_dev,
1725			  "failed to clear extended doorbell info, ret = %d.\n",
1726			  ret);
1727
1728	return ret;
1729}
1730
1731static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
1732{
1733	struct hns_roce_query_fw_info *resp;
1734	struct hns_roce_cmq_desc desc;
1735	int ret;
1736
1737	hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
1738	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1739	if (ret)
1740		return ret;
1741
1742	resp = (struct hns_roce_query_fw_info *)desc.data;
1743	hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
1744
1745	return 0;
1746}
1747
1748static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)
1749{
1750	struct hns_roce_cmq_desc desc;
1751	int ret;
1752
1753	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
1754		hr_dev->func_num = 1;
1755		return 0;
1756	}
1757
1758	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,
1759				      true);
1760	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1761	if (ret) {
1762		hr_dev->func_num = 1;
1763		return ret;
1764	}
1765
1766	hr_dev->func_num = le32_to_cpu(desc.func_info.own_func_num);
1767	hr_dev->cong_algo_tmpl_id = le32_to_cpu(desc.func_info.own_mac_id);
1768
1769	return 0;
1770}
1771
1772static int hns_roce_hw_v2_query_counter(struct hns_roce_dev *hr_dev,
1773					u64 *stats, u32 port, int *num_counters)
1774{
1775#define CNT_PER_DESC 3
1776	struct hns_roce_cmq_desc *desc;
1777	int bd_idx, cnt_idx;
1778	__le64 *cnt_data;
1779	int desc_num;
1780	int ret;
1781	int i;
1782
1783	if (port > hr_dev->caps.num_ports)
1784		return -EINVAL;
1785
1786	desc_num = DIV_ROUND_UP(HNS_ROCE_HW_CNT_TOTAL, CNT_PER_DESC);
1787	desc = kcalloc(desc_num, sizeof(*desc), GFP_KERNEL);
1788	if (!desc)
1789		return -ENOMEM;
1790
1791	for (i = 0; i < desc_num; i++) {
1792		hns_roce_cmq_setup_basic_desc(&desc[i],
1793					      HNS_ROCE_OPC_QUERY_COUNTER, true);
1794		if (i != desc_num - 1)
1795			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1796	}
1797
1798	ret = hns_roce_cmq_send(hr_dev, desc, desc_num);
1799	if (ret) {
1800		ibdev_err(&hr_dev->ib_dev,
1801			  "failed to get counter, ret = %d.\n", ret);
1802		goto err_out;
1803	}
1804
1805	for (i = 0; i < HNS_ROCE_HW_CNT_TOTAL && i < *num_counters; i++) {
1806		bd_idx = i / CNT_PER_DESC;
1807		if (bd_idx != HNS_ROCE_HW_CNT_TOTAL / CNT_PER_DESC &&
1808		    !(desc[bd_idx].flag & cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT)))
1809			break;
1810
1811		cnt_data = (__le64 *)&desc[bd_idx].data[0];
1812		cnt_idx = i % CNT_PER_DESC;
1813		stats[i] = le64_to_cpu(cnt_data[cnt_idx]);
1814	}
1815	*num_counters = i;
1816
1817err_out:
1818	kfree(desc);
1819	return ret;
1820}
1821
1822static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
1823{
1824	struct hns_roce_cmq_desc desc;
1825	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
1826	u32 clock_cycles_of_1us;
1827
1828	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
1829				      false);
1830
1831	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
1832		clock_cycles_of_1us = HNS_ROCE_1NS_CFG;
1833	else
1834		clock_cycles_of_1us = HNS_ROCE_1US_CFG;
1835
1836	hr_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, clock_cycles_of_1us);
1837	hr_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, ROCE_V2_UDP_DPORT);
1838
1839	return hns_roce_cmq_send(hr_dev, &desc, 1);
1840}
1841
1842static int load_func_res_caps(struct hns_roce_dev *hr_dev, bool is_vf)
1843{
1844	struct hns_roce_cmq_desc desc[2];
1845	struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;
1846	struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;
1847	struct hns_roce_caps *caps = &hr_dev->caps;
1848	enum hns_roce_opcode_type opcode;
1849	u32 func_num;
1850	int ret;
1851
1852	if (is_vf) {
1853		opcode = HNS_ROCE_OPC_QUERY_VF_RES;
1854		func_num = 1;
1855	} else {
1856		opcode = HNS_ROCE_OPC_QUERY_PF_RES;
1857		func_num = hr_dev->func_num;
1858	}
1859
1860	hns_roce_cmq_setup_basic_desc(&desc[0], opcode, true);
1861	desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1862	hns_roce_cmq_setup_basic_desc(&desc[1], opcode, true);
1863
1864	ret = hns_roce_cmq_send(hr_dev, desc, 2);
1865	if (ret)
1866		return ret;
1867
1868	caps->qpc_bt_num = hr_reg_read(r_a, FUNC_RES_A_QPC_BT_NUM) / func_num;
1869	caps->srqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_SRQC_BT_NUM) / func_num;
1870	caps->cqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_CQC_BT_NUM) / func_num;
1871	caps->mpt_bt_num = hr_reg_read(r_a, FUNC_RES_A_MPT_BT_NUM) / func_num;
1872	caps->eqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_EQC_BT_NUM) / func_num;
1873	caps->smac_bt_num = hr_reg_read(r_b, FUNC_RES_B_SMAC_NUM) / func_num;
1874	caps->sgid_bt_num = hr_reg_read(r_b, FUNC_RES_B_SGID_NUM) / func_num;
1875	caps->sccc_bt_num = hr_reg_read(r_b, FUNC_RES_B_SCCC_BT_NUM) / func_num;
1876
1877	if (is_vf) {
1878		caps->sl_num = hr_reg_read(r_b, FUNC_RES_V_QID_NUM) / func_num;
1879		caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_V_GMV_BT_NUM) /
1880					       func_num;
1881	} else {
1882		caps->sl_num = hr_reg_read(r_b, FUNC_RES_B_QID_NUM) / func_num;
1883		caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_B_GMV_BT_NUM) /
1884					       func_num;
1885	}
1886
1887	return 0;
1888}
1889
1890static int load_pf_timer_res_caps(struct hns_roce_dev *hr_dev)
1891{
1892	struct hns_roce_cmq_desc desc;
1893	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
1894	struct hns_roce_caps *caps = &hr_dev->caps;
1895	int ret;
1896
1897	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
1898				      true);
1899
1900	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1901	if (ret)
1902		return ret;
1903
1904	caps->qpc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_QPC_ITEM_NUM);
1905	caps->cqc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_CQC_ITEM_NUM);
1906
1907	return 0;
1908}
1909
1910static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
1911{
1912	struct device *dev = hr_dev->dev;
1913	int ret;
1914
1915	ret = load_func_res_caps(hr_dev, false);
1916	if (ret) {
1917		dev_err(dev, "failed to load pf res caps, ret = %d.\n", ret);
1918		return ret;
1919	}
1920
1921	ret = load_pf_timer_res_caps(hr_dev);
1922	if (ret)
1923		dev_err(dev, "failed to load pf timer resource, ret = %d.\n",
1924			ret);
1925
1926	return ret;
1927}
1928
1929static int hns_roce_query_vf_resource(struct hns_roce_dev *hr_dev)
1930{
1931	struct device *dev = hr_dev->dev;
1932	int ret;
1933
1934	ret = load_func_res_caps(hr_dev, true);
1935	if (ret)
1936		dev_err(dev, "failed to load vf res caps, ret = %d.\n", ret);
1937
1938	return ret;
1939}
1940
1941static int __hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,
1942					  u32 vf_id)
1943{
1944	struct hns_roce_vf_switch *swt;
1945	struct hns_roce_cmq_desc desc;
1946	int ret;
1947
1948	swt = (struct hns_roce_vf_switch *)desc.data;
1949	hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
1950	swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
1951	hr_reg_write(swt, VF_SWITCH_VF_ID, vf_id);
1952	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1953	if (ret)
1954		return ret;
1955
1956	desc.flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN);
1957	desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1958	hr_reg_enable(swt, VF_SWITCH_ALW_LPBK);
1959	hr_reg_clear(swt, VF_SWITCH_ALW_LCL_LPBK);
1960	hr_reg_enable(swt, VF_SWITCH_ALW_DST_OVRD);
1961
1962	return hns_roce_cmq_send(hr_dev, &desc, 1);
1963}
1964
1965static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev)
1966{
1967	u32 vf_id;
1968	int ret;
1969
1970	for (vf_id = 0; vf_id < hr_dev->func_num; vf_id++) {
1971		ret = __hns_roce_set_vf_switch_param(hr_dev, vf_id);
1972		if (ret)
1973			return ret;
1974	}
1975	return 0;
1976}
1977
1978static int config_vf_hem_resource(struct hns_roce_dev *hr_dev, int vf_id)
1979{
1980	struct hns_roce_cmq_desc desc[2];
1981	struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;
1982	struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;
1983	enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES;
1984	struct hns_roce_caps *caps = &hr_dev->caps;
1985
1986	hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);
1987	desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1988	hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);
1989
1990	hr_reg_write(r_a, FUNC_RES_A_VF_ID, vf_id);
1991
1992	hr_reg_write(r_a, FUNC_RES_A_QPC_BT_NUM, caps->qpc_bt_num);
1993	hr_reg_write(r_a, FUNC_RES_A_QPC_BT_IDX, vf_id * caps->qpc_bt_num);
1994	hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_NUM, caps->srqc_bt_num);
1995	hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_IDX, vf_id * caps->srqc_bt_num);
1996	hr_reg_write(r_a, FUNC_RES_A_CQC_BT_NUM, caps->cqc_bt_num);
1997	hr_reg_write(r_a, FUNC_RES_A_CQC_BT_IDX, vf_id * caps->cqc_bt_num);
1998	hr_reg_write(r_a, FUNC_RES_A_MPT_BT_NUM, caps->mpt_bt_num);
1999	hr_reg_write(r_a, FUNC_RES_A_MPT_BT_IDX, vf_id * caps->mpt_bt_num);
2000	hr_reg_write(r_a, FUNC_RES_A_EQC_BT_NUM, caps->eqc_bt_num);
2001	hr_reg_write(r_a, FUNC_RES_A_EQC_BT_IDX, vf_id * caps->eqc_bt_num);
2002	hr_reg_write(r_b, FUNC_RES_V_QID_NUM, caps->sl_num);
2003	hr_reg_write(r_b, FUNC_RES_B_QID_IDX, vf_id * caps->sl_num);
2004	hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_NUM, caps->sccc_bt_num);
2005	hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_IDX, vf_id * caps->sccc_bt_num);
2006
2007	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
2008		hr_reg_write(r_b, FUNC_RES_V_GMV_BT_NUM, caps->gmv_bt_num);
2009		hr_reg_write(r_b, FUNC_RES_B_GMV_BT_IDX,
2010			     vf_id * caps->gmv_bt_num);
2011	} else {
2012		hr_reg_write(r_b, FUNC_RES_B_SGID_NUM, caps->sgid_bt_num);
2013		hr_reg_write(r_b, FUNC_RES_B_SGID_IDX,
2014			     vf_id * caps->sgid_bt_num);
2015		hr_reg_write(r_b, FUNC_RES_B_SMAC_NUM, caps->smac_bt_num);
2016		hr_reg_write(r_b, FUNC_RES_B_SMAC_IDX,
2017			     vf_id * caps->smac_bt_num);
2018	}
2019
2020	return hns_roce_cmq_send(hr_dev, desc, 2);
2021}
2022
2023static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
2024{
2025	u32 func_num = max_t(u32, 1, hr_dev->func_num);
2026	u32 vf_id;
2027	int ret;
2028
2029	for (vf_id = 0; vf_id < func_num; vf_id++) {
2030		ret = config_vf_hem_resource(hr_dev, vf_id);
2031		if (ret) {
2032			dev_err(hr_dev->dev,
2033				"failed to config vf-%u hem res, ret = %d.\n",
2034				vf_id, ret);
2035			return ret;
2036		}
2037	}
2038
2039	return 0;
2040}
2041
2042static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
2043{
2044	struct hns_roce_cmq_desc desc;
2045	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
2046	struct hns_roce_caps *caps = &hr_dev->caps;
2047
2048	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
2049
2050	hr_reg_write(req, CFG_BT_ATTR_QPC_BA_PGSZ,
2051		     caps->qpc_ba_pg_sz + PG_SHIFT_OFFSET);
2052	hr_reg_write(req, CFG_BT_ATTR_QPC_BUF_PGSZ,
2053		     caps->qpc_buf_pg_sz + PG_SHIFT_OFFSET);
2054	hr_reg_write(req, CFG_BT_ATTR_QPC_HOPNUM,
2055		     to_hr_hem_hopnum(caps->qpc_hop_num, caps->num_qps));
2056
2057	hr_reg_write(req, CFG_BT_ATTR_SRQC_BA_PGSZ,
2058		     caps->srqc_ba_pg_sz + PG_SHIFT_OFFSET);
2059	hr_reg_write(req, CFG_BT_ATTR_SRQC_BUF_PGSZ,
2060		     caps->srqc_buf_pg_sz + PG_SHIFT_OFFSET);
2061	hr_reg_write(req, CFG_BT_ATTR_SRQC_HOPNUM,
2062		     to_hr_hem_hopnum(caps->srqc_hop_num, caps->num_srqs));
2063
2064	hr_reg_write(req, CFG_BT_ATTR_CQC_BA_PGSZ,
2065		     caps->cqc_ba_pg_sz + PG_SHIFT_OFFSET);
2066	hr_reg_write(req, CFG_BT_ATTR_CQC_BUF_PGSZ,
2067		     caps->cqc_buf_pg_sz + PG_SHIFT_OFFSET);
2068	hr_reg_write(req, CFG_BT_ATTR_CQC_HOPNUM,
2069		     to_hr_hem_hopnum(caps->cqc_hop_num, caps->num_cqs));
2070
2071	hr_reg_write(req, CFG_BT_ATTR_MPT_BA_PGSZ,
2072		     caps->mpt_ba_pg_sz + PG_SHIFT_OFFSET);
2073	hr_reg_write(req, CFG_BT_ATTR_MPT_BUF_PGSZ,
2074		     caps->mpt_buf_pg_sz + PG_SHIFT_OFFSET);
2075	hr_reg_write(req, CFG_BT_ATTR_MPT_HOPNUM,
2076		     to_hr_hem_hopnum(caps->mpt_hop_num, caps->num_mtpts));
2077
2078	hr_reg_write(req, CFG_BT_ATTR_SCCC_BA_PGSZ,
2079		     caps->sccc_ba_pg_sz + PG_SHIFT_OFFSET);
2080	hr_reg_write(req, CFG_BT_ATTR_SCCC_BUF_PGSZ,
2081		     caps->sccc_buf_pg_sz + PG_SHIFT_OFFSET);
2082	hr_reg_write(req, CFG_BT_ATTR_SCCC_HOPNUM,
2083		     to_hr_hem_hopnum(caps->sccc_hop_num, caps->num_qps));
2084
2085	return hns_roce_cmq_send(hr_dev, &desc, 1);
2086}
2087
2088static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num,
2089		       u32 *buf_page_size, u32 *bt_page_size, u32 hem_type)
2090{
2091	u64 obj_per_chunk;
2092	u64 bt_chunk_size = PAGE_SIZE;
2093	u64 buf_chunk_size = PAGE_SIZE;
2094	u64 obj_per_chunk_default = buf_chunk_size / obj_size;
2095
2096	*buf_page_size = 0;
2097	*bt_page_size = 0;
2098
2099	switch (hop_num) {
2100	case 3:
2101		obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
2102				(bt_chunk_size / BA_BYTE_LEN) *
2103				(bt_chunk_size / BA_BYTE_LEN) *
2104				 obj_per_chunk_default;
2105		break;
2106	case 2:
2107		obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
2108				(bt_chunk_size / BA_BYTE_LEN) *
2109				 obj_per_chunk_default;
2110		break;
2111	case 1:
2112		obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
2113				obj_per_chunk_default;
2114		break;
2115	case HNS_ROCE_HOP_NUM_0:
2116		obj_per_chunk = ctx_bt_num * obj_per_chunk_default;
2117		break;
2118	default:
2119		pr_err("table %u not support hop_num = %u!\n", hem_type,
2120		       hop_num);
2121		return;
2122	}
2123
2124	if (hem_type >= HEM_TYPE_MTT)
2125		*bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
2126	else
2127		*buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
2128}
2129
2130static void set_hem_page_size(struct hns_roce_dev *hr_dev)
2131{
2132	struct hns_roce_caps *caps = &hr_dev->caps;
2133
2134	/* EQ */
2135	caps->eqe_ba_pg_sz = 0;
2136	caps->eqe_buf_pg_sz = 0;
2137
2138	/* Link Table */
2139	caps->llm_buf_pg_sz = 0;
2140
2141	/* MR */
2142	caps->mpt_ba_pg_sz = 0;
2143	caps->mpt_buf_pg_sz = 0;
2144	caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;
2145	caps->pbl_buf_pg_sz = 0;
2146	calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num,
2147		   caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz,
2148		   HEM_TYPE_MTPT);
2149
2150	/* QP */
2151	caps->qpc_ba_pg_sz = 0;
2152	caps->qpc_buf_pg_sz = 0;
2153	caps->qpc_timer_ba_pg_sz = 0;
2154	caps->qpc_timer_buf_pg_sz = 0;
2155	caps->sccc_ba_pg_sz = 0;
2156	caps->sccc_buf_pg_sz = 0;
2157	caps->mtt_ba_pg_sz = 0;
2158	caps->mtt_buf_pg_sz = 0;
2159	calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num,
2160		   caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz,
2161		   HEM_TYPE_QPC);
2162
2163	if (caps->flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
2164		calc_pg_sz(caps->num_qps, caps->sccc_sz, caps->sccc_hop_num,
2165			   caps->sccc_bt_num, &caps->sccc_buf_pg_sz,
2166			   &caps->sccc_ba_pg_sz, HEM_TYPE_SCCC);
2167
2168	/* CQ */
2169	caps->cqc_ba_pg_sz = 0;
2170	caps->cqc_buf_pg_sz = 0;
2171	caps->cqc_timer_ba_pg_sz = 0;
2172	caps->cqc_timer_buf_pg_sz = 0;
2173	caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K;
2174	caps->cqe_buf_pg_sz = 0;
2175	calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num,
2176		   caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz,
2177		   HEM_TYPE_CQC);
2178	calc_pg_sz(caps->max_cqes, caps->cqe_sz, caps->cqe_hop_num,
2179		   1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);
2180
2181	/* SRQ */
2182	if (caps->flags & HNS_ROCE_CAP_FLAG_SRQ) {
2183		caps->srqc_ba_pg_sz = 0;
2184		caps->srqc_buf_pg_sz = 0;
2185		caps->srqwqe_ba_pg_sz = 0;
2186		caps->srqwqe_buf_pg_sz = 0;
2187		caps->idx_ba_pg_sz = 0;
2188		caps->idx_buf_pg_sz = 0;
2189		calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz,
2190			   caps->srqc_hop_num, caps->srqc_bt_num,
2191			   &caps->srqc_buf_pg_sz, &caps->srqc_ba_pg_sz,
2192			   HEM_TYPE_SRQC);
2193		calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz,
2194			   caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz,
2195			   &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE);
2196		calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz,
2197			   caps->idx_hop_num, 1, &caps->idx_buf_pg_sz,
2198			   &caps->idx_ba_pg_sz, HEM_TYPE_IDX);
2199	}
2200
2201	/* GMV */
2202	caps->gmv_ba_pg_sz = 0;
2203	caps->gmv_buf_pg_sz = 0;
2204}
2205
2206/* Apply all loaded caps before setting to hardware */
2207static void apply_func_caps(struct hns_roce_dev *hr_dev)
2208{
2209#define MAX_GID_TBL_LEN 256
2210	struct hns_roce_caps *caps = &hr_dev->caps;
2211	struct hns_roce_v2_priv *priv = hr_dev->priv;
2212
2213	/* The following configurations don't need to be got from firmware. */
2214	caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
2215	caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
2216	caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
2217
2218	caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
2219	caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2220	caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2221
2222	caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
2223	caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
2224
2225	if (!caps->num_comp_vectors)
2226		caps->num_comp_vectors =
2227			min_t(u32, caps->eqc_bt_num - HNS_ROCE_V2_AEQE_VEC_NUM,
2228				(u32)priv->handle->rinfo.num_vectors -
2229		(HNS_ROCE_V2_AEQE_VEC_NUM + HNS_ROCE_V2_ABNORMAL_VEC_NUM));
2230
2231	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
2232		caps->eqe_hop_num = HNS_ROCE_V3_EQE_HOP_NUM;
2233		caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
2234		caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;
2235
2236		/* The following configurations will be overwritten */
2237		caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
2238		caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
2239		caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;
2240
2241		/* The following configurations are not got from firmware */
2242		caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;
2243
2244		caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0;
2245
2246		/* It's meaningless to support excessively large gid_table_len,
2247		 * as the type of sgid_index in kernel struct ib_global_route
2248		 * and userspace struct ibv_global_route are u8/uint8_t (0-255).
2249		 */
2250		caps->gid_table_len[0] = min_t(u32, MAX_GID_TBL_LEN,
2251					 caps->gmv_bt_num *
2252					 (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz));
2253
2254		caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE /
2255							  caps->gmv_entry_sz);
2256	} else {
2257		u32 func_num = max_t(u32, 1, hr_dev->func_num);
2258
2259		caps->eqe_hop_num = HNS_ROCE_V2_EQE_HOP_NUM;
2260		caps->ceqe_size = HNS_ROCE_CEQE_SIZE;
2261		caps->aeqe_size = HNS_ROCE_AEQE_SIZE;
2262		caps->gid_table_len[0] /= func_num;
2263	}
2264
2265	if (hr_dev->is_vf) {
2266		caps->default_aeq_arm_st = 0x3;
2267		caps->default_ceq_arm_st = 0x3;
2268		caps->default_ceq_max_cnt = 0x1;
2269		caps->default_ceq_period = 0x10;
2270		caps->default_aeq_max_cnt = 0x1;
2271		caps->default_aeq_period = 0x10;
2272	}
2273
2274	set_hem_page_size(hr_dev);
2275}
2276
2277static int hns_roce_query_caps(struct hns_roce_dev *hr_dev)
2278{
2279	struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM] = {};
2280	struct hns_roce_caps *caps = &hr_dev->caps;
2281	struct hns_roce_query_pf_caps_a *resp_a;
2282	struct hns_roce_query_pf_caps_b *resp_b;
2283	struct hns_roce_query_pf_caps_c *resp_c;
2284	struct hns_roce_query_pf_caps_d *resp_d;
2285	struct hns_roce_query_pf_caps_e *resp_e;
2286	struct hns_roce_query_pf_caps_f *resp_f;
2287	enum hns_roce_opcode_type cmd;
2288	int ctx_hop_num;
2289	int pbl_hop_num;
2290	int cmd_num;
2291	int ret;
2292	int i;
2293
2294	cmd = hr_dev->is_vf ? HNS_ROCE_OPC_QUERY_VF_CAPS_NUM :
2295	      HNS_ROCE_OPC_QUERY_PF_CAPS_NUM;
2296	cmd_num = hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ?
2297		  HNS_ROCE_QUERY_PF_CAPS_CMD_NUM_HIP08 :
2298		  HNS_ROCE_QUERY_PF_CAPS_CMD_NUM;
2299
2300	for (i = 0; i < cmd_num - 1; i++) {
2301		hns_roce_cmq_setup_basic_desc(&desc[i], cmd, true);
2302		desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2303	}
2304
2305	hns_roce_cmq_setup_basic_desc(&desc[cmd_num - 1], cmd, true);
2306	desc[cmd_num - 1].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2307
2308	ret = hns_roce_cmq_send(hr_dev, desc, cmd_num);
2309	if (ret)
2310		return ret;
2311
2312	resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data;
2313	resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data;
2314	resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;
2315	resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;
2316	resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;
2317	resp_f = (struct hns_roce_query_pf_caps_f *)desc[5].data;
2318
2319	caps->local_ca_ack_delay = resp_a->local_ca_ack_delay;
2320	caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg);
2321	caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline);
2322	caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg);
2323	caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg);
2324	caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges);
2325	caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges);
2326	caps->num_aeq_vectors = resp_a->num_aeq_vectors;
2327	caps->num_other_vectors = resp_a->num_other_vectors;
2328	caps->max_sq_desc_sz = resp_a->max_sq_desc_sz;
2329	caps->max_rq_desc_sz = resp_a->max_rq_desc_sz;
2330
2331	caps->mtpt_entry_sz = resp_b->mtpt_entry_sz;
2332	caps->irrl_entry_sz = resp_b->irrl_entry_sz;
2333	caps->trrl_entry_sz = resp_b->trrl_entry_sz;
2334	caps->cqc_entry_sz = resp_b->cqc_entry_sz;
2335	caps->srqc_entry_sz = resp_b->srqc_entry_sz;
2336	caps->idx_entry_sz = resp_b->idx_entry_sz;
2337	caps->sccc_sz = resp_b->sccc_sz;
2338	caps->max_mtu = resp_b->max_mtu;
2339	caps->min_cqes = resp_b->min_cqes;
2340	caps->min_wqes = resp_b->min_wqes;
2341	caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap);
2342	caps->pkey_table_len[0] = resp_b->pkey_table_len;
2343	caps->phy_num_uars = resp_b->phy_num_uars;
2344	ctx_hop_num = resp_b->ctx_hop_num;
2345	pbl_hop_num = resp_b->pbl_hop_num;
2346
2347	caps->num_pds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_PDS);
2348
2349	caps->flags = hr_reg_read(resp_c, PF_CAPS_C_CAP_FLAGS);
2350	caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) <<
2351		       HNS_ROCE_CAP_FLAGS_EX_SHIFT;
2352
2353	if (hr_dev->is_vf)
2354		caps->flags &= ~HNS_ROCE_CAP_FLAG_BOND;
2355
2356	caps->num_cqs = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_CQS);
2357	caps->gid_table_len[0] = hr_reg_read(resp_c, PF_CAPS_C_MAX_GID);
2358	caps->max_cqes = 1 << hr_reg_read(resp_c, PF_CAPS_C_CQ_DEPTH);
2359	caps->num_xrcds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_XRCDS);
2360	caps->num_mtpts = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_MRWS);
2361	caps->num_qps = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_QPS);
2362	caps->max_qp_init_rdma = hr_reg_read(resp_c, PF_CAPS_C_MAX_ORD);
2363	caps->max_qp_dest_rdma = caps->max_qp_init_rdma;
2364	caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth);
2365
2366	caps->num_srqs = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_SRQS);
2367	caps->cong_cap = hr_reg_read(resp_d, PF_CAPS_D_CONG_CAP);
2368	caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth);
2369	caps->ceqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_CEQ_DEPTH);
2370	caps->num_comp_vectors = hr_reg_read(resp_d, PF_CAPS_D_NUM_CEQS);
2371	caps->aeqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_AEQ_DEPTH);
2372	caps->default_cong_type = hr_reg_read(resp_d, PF_CAPS_D_DEFAULT_ALG);
2373	caps->reserved_pds = hr_reg_read(resp_d, PF_CAPS_D_RSV_PDS);
2374	caps->num_uars = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_UARS);
2375	caps->reserved_qps = hr_reg_read(resp_d, PF_CAPS_D_RSV_QPS);
2376	caps->reserved_uars = hr_reg_read(resp_d, PF_CAPS_D_RSV_UARS);
2377
2378	caps->reserved_mrws = hr_reg_read(resp_e, PF_CAPS_E_RSV_MRWS);
2379	caps->chunk_sz = 1 << hr_reg_read(resp_e, PF_CAPS_E_CHUNK_SIZE_SHIFT);
2380	caps->reserved_cqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_CQS);
2381	caps->reserved_xrcds = hr_reg_read(resp_e, PF_CAPS_E_RSV_XRCDS);
2382	caps->reserved_srqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_SRQS);
2383	caps->reserved_lkey = hr_reg_read(resp_e, PF_CAPS_E_RSV_LKEYS);
2384
2385	caps->max_ack_req_msg_len = le32_to_cpu(resp_f->max_ack_req_msg_len);
2386
2387	caps->qpc_hop_num = ctx_hop_num;
2388	caps->sccc_hop_num = ctx_hop_num;
2389	caps->srqc_hop_num = ctx_hop_num;
2390	caps->cqc_hop_num = ctx_hop_num;
2391	caps->mpt_hop_num = ctx_hop_num;
2392	caps->mtt_hop_num = pbl_hop_num;
2393	caps->cqe_hop_num = pbl_hop_num;
2394	caps->srqwqe_hop_num = pbl_hop_num;
2395	caps->idx_hop_num = pbl_hop_num;
2396	caps->wqe_sq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_SQWQE_HOP_NUM);
2397	caps->wqe_sge_hop_num = hr_reg_read(resp_d, PF_CAPS_D_EX_SGE_HOP_NUM);
2398	caps->wqe_rq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_RQWQE_HOP_NUM);
2399
2400	if (!(caps->page_size_cap & PAGE_SIZE))
2401		caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
2402
2403	if (!hr_dev->is_vf) {
2404		caps->cqe_sz = resp_a->cqe_sz;
2405		caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz);
2406		caps->default_aeq_arm_st =
2407				hr_reg_read(resp_d, PF_CAPS_D_AEQ_ARM_ST);
2408		caps->default_ceq_arm_st =
2409				hr_reg_read(resp_d, PF_CAPS_D_CEQ_ARM_ST);
2410		caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);
2411		caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);
2412		caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);
2413		caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);
2414	}
2415
2416	return 0;
2417}
2418
2419static int config_hem_entry_size(struct hns_roce_dev *hr_dev, u32 type, u32 val)
2420{
2421	struct hns_roce_cmq_desc desc;
2422	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
2423
2424	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,
2425				      false);
2426
2427	hr_reg_write(req, CFG_HEM_ENTRY_SIZE_TYPE, type);
2428	hr_reg_write(req, CFG_HEM_ENTRY_SIZE_VALUE, val);
2429
2430	return hns_roce_cmq_send(hr_dev, &desc, 1);
2431}
2432
2433static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev)
2434{
2435	struct hns_roce_caps *caps = &hr_dev->caps;
2436	int ret;
2437
2438	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
2439		return 0;
2440
2441	ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE,
2442				    caps->qpc_sz);
2443	if (ret) {
2444		dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret);
2445		return ret;
2446	}
2447
2448	ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_SCCC_SIZE,
2449				    caps->sccc_sz);
2450	if (ret)
2451		dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret);
2452
2453	return ret;
2454}
2455
2456static int hns_roce_v2_vf_profile(struct hns_roce_dev *hr_dev)
2457{
2458	struct device *dev = hr_dev->dev;
2459	int ret;
2460
2461	hr_dev->func_num = 1;
2462
2463	ret = hns_roce_query_caps(hr_dev);
2464	if (ret) {
2465		dev_err(dev, "failed to query VF caps, ret = %d.\n", ret);
2466		return ret;
2467	}
2468
2469	ret = hns_roce_query_vf_resource(hr_dev);
2470	if (ret) {
2471		dev_err(dev, "failed to query VF resource, ret = %d.\n", ret);
2472		return ret;
2473	}
2474
2475	apply_func_caps(hr_dev);
2476
2477	ret = hns_roce_v2_set_bt(hr_dev);
2478	if (ret)
2479		dev_err(dev, "failed to config VF BA table, ret = %d.\n", ret);
2480
2481	return ret;
2482}
2483
2484static int hns_roce_v2_pf_profile(struct hns_roce_dev *hr_dev)
2485{
2486	struct device *dev = hr_dev->dev;
2487	int ret;
2488
2489	ret = hns_roce_query_func_info(hr_dev);
2490	if (ret) {
2491		dev_err(dev, "failed to query func info, ret = %d.\n", ret);
2492		return ret;
2493	}
2494
2495	ret = hns_roce_config_global_param(hr_dev);
2496	if (ret) {
2497		dev_err(dev, "failed to config global param, ret = %d.\n", ret);
2498		return ret;
2499	}
2500
2501	ret = hns_roce_set_vf_switch_param(hr_dev);
2502	if (ret) {
2503		dev_err(dev, "failed to set switch param, ret = %d.\n", ret);
2504		return ret;
2505	}
2506
2507	ret = hns_roce_query_caps(hr_dev);
2508	if (ret) {
2509		dev_err(dev, "failed to query PF caps, ret = %d.\n", ret);
2510		return ret;
2511	}
2512
2513	ret = hns_roce_query_pf_resource(hr_dev);
2514	if (ret) {
2515		dev_err(dev, "failed to query pf resource, ret = %d.\n", ret);
2516		return ret;
2517	}
2518
2519	apply_func_caps(hr_dev);
2520
2521	ret = hns_roce_alloc_vf_resource(hr_dev);
2522	if (ret) {
2523		dev_err(dev, "failed to alloc vf resource, ret = %d.\n", ret);
2524		return ret;
2525	}
2526
2527	ret = hns_roce_v2_set_bt(hr_dev);
2528	if (ret) {
2529		dev_err(dev, "failed to config BA table, ret = %d.\n", ret);
2530		return ret;
2531	}
2532
2533	/* Configure the size of QPC, SCCC, etc. */
2534	return hns_roce_config_entry_size(hr_dev);
2535}
2536
2537static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
2538{
2539	struct device *dev = hr_dev->dev;
2540	int ret;
2541
2542	ret = hns_roce_cmq_query_hw_info(hr_dev);
2543	if (ret) {
2544		dev_err(dev, "failed to query hardware info, ret = %d.\n", ret);
2545		return ret;
2546	}
2547
2548	ret = hns_roce_query_fw_ver(hr_dev);
2549	if (ret) {
2550		dev_err(dev, "failed to query firmware info, ret = %d.\n", ret);
2551		return ret;
2552	}
2553
2554	hr_dev->vendor_part_id = hr_dev->pci_dev->device;
2555	hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
2556
2557	if (hr_dev->is_vf)
2558		return hns_roce_v2_vf_profile(hr_dev);
2559	else
2560		return hns_roce_v2_pf_profile(hr_dev);
2561}
2562
2563static void config_llm_table(struct hns_roce_buf *data_buf, void *cfg_buf)
2564{
2565	u32 i, next_ptr, page_num;
2566	__le64 *entry = cfg_buf;
2567	dma_addr_t addr;
2568	u64 val;
2569
2570	page_num = data_buf->npages;
2571	for (i = 0; i < page_num; i++) {
2572		addr = hns_roce_buf_page(data_buf, i);
2573		if (i == (page_num - 1))
2574			next_ptr = 0;
2575		else
2576			next_ptr = i + 1;
2577
2578		val = HNS_ROCE_EXT_LLM_ENTRY(addr, (u64)next_ptr);
2579		entry[i] = cpu_to_le64(val);
2580	}
2581}
2582
2583static int set_llm_cfg_to_hw(struct hns_roce_dev *hr_dev,
2584			     struct hns_roce_link_table *table)
2585{
2586	struct hns_roce_cmq_desc desc[2];
2587	struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;
2588	struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;
2589	struct hns_roce_buf *buf = table->buf;
2590	enum hns_roce_opcode_type opcode;
2591	dma_addr_t addr;
2592
2593	opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
2594	hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);
2595	desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2596	hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);
2597
2598	hr_reg_write(r_a, CFG_LLM_A_BA_L, lower_32_bits(table->table.map));
2599	hr_reg_write(r_a, CFG_LLM_A_BA_H, upper_32_bits(table->table.map));
2600	hr_reg_write(r_a, CFG_LLM_A_DEPTH, buf->npages);
2601	hr_reg_write(r_a, CFG_LLM_A_PGSZ, to_hr_hw_page_shift(buf->page_shift));
2602	hr_reg_enable(r_a, CFG_LLM_A_INIT_EN);
2603
2604	addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, 0));
2605	hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_L, lower_32_bits(addr));
2606	hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_H, upper_32_bits(addr));
2607	hr_reg_write(r_a, CFG_LLM_A_HEAD_NXTPTR, 1);
2608	hr_reg_write(r_a, CFG_LLM_A_HEAD_PTR, 0);
2609
2610	addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, buf->npages - 1));
2611	hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_L, lower_32_bits(addr));
2612	hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_H, upper_32_bits(addr));
2613	hr_reg_write(r_b, CFG_LLM_B_TAIL_PTR, buf->npages - 1);
2614
2615	return hns_roce_cmq_send(hr_dev, desc, 2);
2616}
2617
2618static struct hns_roce_link_table *
2619alloc_link_table_buf(struct hns_roce_dev *hr_dev)
2620{
2621	u16 total_sl = hr_dev->caps.sl_num * hr_dev->func_num;
2622	struct hns_roce_v2_priv *priv = hr_dev->priv;
2623	struct hns_roce_link_table *link_tbl;
2624	u32 pg_shift, size, min_size;
2625
2626	link_tbl = &priv->ext_llm;
2627	pg_shift = hr_dev->caps.llm_buf_pg_sz + PAGE_SHIFT;
2628	size = hr_dev->caps.num_qps * hr_dev->func_num *
2629	       HNS_ROCE_V2_EXT_LLM_ENTRY_SZ;
2630	min_size = HNS_ROCE_EXT_LLM_MIN_PAGES(total_sl) << pg_shift;
2631
2632	/* Alloc data table */
2633	size = max(size, min_size);
2634	link_tbl->buf = hns_roce_buf_alloc(hr_dev, size, pg_shift, 0);
2635	if (IS_ERR(link_tbl->buf))
2636		return ERR_PTR(-ENOMEM);
2637
2638	/* Alloc config table */
2639	size = link_tbl->buf->npages * sizeof(u64);
2640	link_tbl->table.buf = dma_alloc_coherent(hr_dev->dev, size,
2641						 &link_tbl->table.map,
2642						 GFP_KERNEL);
2643	if (!link_tbl->table.buf) {
2644		hns_roce_buf_free(hr_dev, link_tbl->buf);
2645		return ERR_PTR(-ENOMEM);
2646	}
2647
2648	return link_tbl;
2649}
2650
2651static void free_link_table_buf(struct hns_roce_dev *hr_dev,
2652				struct hns_roce_link_table *tbl)
2653{
2654	if (tbl->buf) {
2655		u32 size = tbl->buf->npages * sizeof(u64);
2656
2657		dma_free_coherent(hr_dev->dev, size, tbl->table.buf,
2658				  tbl->table.map);
2659	}
2660
2661	hns_roce_buf_free(hr_dev, tbl->buf);
2662}
2663
2664static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev)
2665{
2666	struct hns_roce_link_table *link_tbl;
2667	int ret;
2668
2669	link_tbl = alloc_link_table_buf(hr_dev);
2670	if (IS_ERR(link_tbl))
2671		return -ENOMEM;
2672
2673	if (WARN_ON(link_tbl->buf->npages > HNS_ROCE_V2_EXT_LLM_MAX_DEPTH)) {
2674		ret = -EINVAL;
2675		goto err_alloc;
2676	}
2677
2678	config_llm_table(link_tbl->buf, link_tbl->table.buf);
2679	ret = set_llm_cfg_to_hw(hr_dev, link_tbl);
2680	if (ret)
2681		goto err_alloc;
2682
2683	return 0;
2684
2685err_alloc:
2686	free_link_table_buf(hr_dev, link_tbl);
2687	return ret;
2688}
2689
2690static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev)
2691{
2692	struct hns_roce_v2_priv *priv = hr_dev->priv;
2693
2694	free_link_table_buf(hr_dev, &priv->ext_llm);
2695}
2696
2697static void free_dip_entry(struct hns_roce_dev *hr_dev)
2698{
2699	struct hns_roce_dip *hr_dip;
2700	unsigned long idx;
2701
2702	xa_lock(&hr_dev->qp_table.dip_xa);
2703
2704	xa_for_each(&hr_dev->qp_table.dip_xa, idx, hr_dip) {
2705		__xa_erase(&hr_dev->qp_table.dip_xa, hr_dip->dip_idx);
2706		kfree(hr_dip);
2707	}
2708
2709	xa_unlock(&hr_dev->qp_table.dip_xa);
2710}
2711
2712static struct ib_pd *free_mr_init_pd(struct hns_roce_dev *hr_dev)
2713{
2714	struct hns_roce_v2_priv *priv = hr_dev->priv;
2715	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2716	struct ib_device *ibdev = &hr_dev->ib_dev;
2717	struct hns_roce_pd *hr_pd;
2718	struct ib_pd *pd;
2719
2720	hr_pd = kzalloc(sizeof(*hr_pd), GFP_KERNEL);
2721	if (!hr_pd)
2722		return NULL;
2723	pd = &hr_pd->ibpd;
2724	pd->device = ibdev;
2725
2726	if (hns_roce_alloc_pd(pd, NULL)) {
2727		ibdev_err(ibdev, "failed to create pd for free mr.\n");
2728		kfree(hr_pd);
2729		return NULL;
2730	}
2731	free_mr->rsv_pd = to_hr_pd(pd);
2732	free_mr->rsv_pd->ibpd.device = &hr_dev->ib_dev;
2733	free_mr->rsv_pd->ibpd.uobject = NULL;
2734	free_mr->rsv_pd->ibpd.__internal_mr = NULL;
2735	atomic_set(&free_mr->rsv_pd->ibpd.usecnt, 0);
2736
2737	return pd;
2738}
2739
2740static struct ib_cq *free_mr_init_cq(struct hns_roce_dev *hr_dev)
2741{
2742	struct hns_roce_v2_priv *priv = hr_dev->priv;
2743	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2744	struct ib_device *ibdev = &hr_dev->ib_dev;
2745	struct ib_cq_init_attr cq_init_attr = {};
2746	struct hns_roce_cq *hr_cq;
2747	struct ib_cq *cq;
2748
2749	cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM;
2750
2751	hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL);
2752	if (!hr_cq)
2753		return NULL;
2754
2755	cq = &hr_cq->ib_cq;
2756	cq->device = ibdev;
2757
2758	if (hns_roce_create_cq(cq, &cq_init_attr, NULL)) {
2759		ibdev_err(ibdev, "failed to create cq for free mr.\n");
2760		kfree(hr_cq);
2761		return NULL;
2762	}
2763	free_mr->rsv_cq = to_hr_cq(cq);
2764	free_mr->rsv_cq->ib_cq.device = &hr_dev->ib_dev;
2765	free_mr->rsv_cq->ib_cq.uobject = NULL;
2766	free_mr->rsv_cq->ib_cq.comp_handler = NULL;
2767	free_mr->rsv_cq->ib_cq.event_handler = NULL;
2768	free_mr->rsv_cq->ib_cq.cq_context = NULL;
2769	atomic_set(&free_mr->rsv_cq->ib_cq.usecnt, 0);
2770
2771	return cq;
2772}
2773
2774static int free_mr_init_qp(struct hns_roce_dev *hr_dev, struct ib_cq *cq,
2775			   struct ib_qp_init_attr *init_attr, int i)
2776{
2777	struct hns_roce_v2_priv *priv = hr_dev->priv;
2778	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2779	struct ib_device *ibdev = &hr_dev->ib_dev;
2780	struct hns_roce_qp *hr_qp;
2781	struct ib_qp *qp;
2782	int ret;
2783
2784	hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
2785	if (!hr_qp)
2786		return -ENOMEM;
2787
2788	qp = &hr_qp->ibqp;
2789	qp->device = ibdev;
2790
2791	ret = hns_roce_create_qp(qp, init_attr, NULL);
2792	if (ret) {
2793		ibdev_err(ibdev, "failed to create qp for free mr.\n");
2794		kfree(hr_qp);
2795		return ret;
2796	}
2797
2798	free_mr->rsv_qp[i] = hr_qp;
2799	free_mr->rsv_qp[i]->ibqp.recv_cq = cq;
2800	free_mr->rsv_qp[i]->ibqp.send_cq = cq;
2801
2802	return 0;
2803}
2804
2805static void free_mr_exit(struct hns_roce_dev *hr_dev)
2806{
2807	struct hns_roce_v2_priv *priv = hr_dev->priv;
2808	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2809	struct ib_qp *qp;
2810	int i;
2811
2812	for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
2813		if (free_mr->rsv_qp[i]) {
2814			qp = &free_mr->rsv_qp[i]->ibqp;
2815			hns_roce_v2_destroy_qp(qp, NULL);
2816			kfree(free_mr->rsv_qp[i]);
2817			free_mr->rsv_qp[i] = NULL;
2818		}
2819	}
2820
2821	if (free_mr->rsv_cq) {
2822		hns_roce_destroy_cq(&free_mr->rsv_cq->ib_cq, NULL);
2823		kfree(free_mr->rsv_cq);
2824		free_mr->rsv_cq = NULL;
2825	}
2826
2827	if (free_mr->rsv_pd) {
2828		hns_roce_dealloc_pd(&free_mr->rsv_pd->ibpd, NULL);
2829		kfree(free_mr->rsv_pd);
2830		free_mr->rsv_pd = NULL;
2831	}
2832
2833	mutex_destroy(&free_mr->mutex);
2834}
2835
2836static int free_mr_alloc_res(struct hns_roce_dev *hr_dev)
2837{
2838	struct hns_roce_v2_priv *priv = hr_dev->priv;
2839	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2840	struct ib_qp_init_attr qp_init_attr = {};
2841	struct ib_pd *pd;
2842	struct ib_cq *cq;
2843	int ret;
2844	int i;
2845
2846	pd = free_mr_init_pd(hr_dev);
2847	if (!pd)
2848		return -ENOMEM;
2849
2850	cq = free_mr_init_cq(hr_dev);
2851	if (!cq) {
2852		ret = -ENOMEM;
2853		goto create_failed_cq;
2854	}
2855
2856	qp_init_attr.qp_type = IB_QPT_RC;
2857	qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
2858	qp_init_attr.send_cq = cq;
2859	qp_init_attr.recv_cq = cq;
2860	for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
2861		qp_init_attr.cap.max_send_wr = HNS_ROCE_FREE_MR_USED_SQWQE_NUM;
2862		qp_init_attr.cap.max_send_sge = HNS_ROCE_FREE_MR_USED_SQSGE_NUM;
2863		qp_init_attr.cap.max_recv_wr = HNS_ROCE_FREE_MR_USED_RQWQE_NUM;
2864		qp_init_attr.cap.max_recv_sge = HNS_ROCE_FREE_MR_USED_RQSGE_NUM;
2865
2866		ret = free_mr_init_qp(hr_dev, cq, &qp_init_attr, i);
2867		if (ret)
2868			goto create_failed_qp;
2869	}
2870
2871	return 0;
2872
2873create_failed_qp:
2874	for (i--; i >= 0; i--) {
2875		hns_roce_v2_destroy_qp(&free_mr->rsv_qp[i]->ibqp, NULL);
2876		kfree(free_mr->rsv_qp[i]);
2877	}
2878	hns_roce_destroy_cq(cq, NULL);
2879	kfree(cq);
2880
2881create_failed_cq:
2882	hns_roce_dealloc_pd(pd, NULL);
2883	kfree(pd);
2884
2885	return ret;
2886}
2887
2888static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev,
2889				 struct ib_qp_attr *attr, int sl_num)
2890{
2891	struct hns_roce_v2_priv *priv = hr_dev->priv;
2892	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2893	struct ib_device *ibdev = &hr_dev->ib_dev;
2894	struct hns_roce_qp *hr_qp;
2895	int loopback;
2896	int mask;
2897	int ret;
2898
2899	hr_qp = to_hr_qp(&free_mr->rsv_qp[sl_num]->ibqp);
2900	hr_qp->free_mr_en = 1;
2901	hr_qp->ibqp.device = ibdev;
2902	hr_qp->ibqp.qp_type = IB_QPT_RC;
2903
2904	mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS;
2905	attr->qp_state = IB_QPS_INIT;
2906	attr->port_num = 1;
2907	attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
2908	ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
2909				    IB_QPS_INIT, NULL);
2910	if (ret) {
2911		ibdev_err_ratelimited(ibdev, "failed to modify qp to init, ret = %d.\n",
2912				      ret);
2913		return ret;
2914	}
2915
2916	loopback = hr_dev->loop_idc;
2917	/* Set qpc lbi = 1 incidate loopback IO */
2918	hr_dev->loop_idc = 1;
2919
2920	mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN |
2921	       IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER;
2922	attr->qp_state = IB_QPS_RTR;
2923	attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2924	attr->path_mtu = IB_MTU_256;
2925	attr->dest_qp_num = hr_qp->qpn;
2926	attr->rq_psn = HNS_ROCE_FREE_MR_USED_PSN;
2927
2928	rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num);
2929
2930	ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
2931				    IB_QPS_RTR, NULL);
2932	hr_dev->loop_idc = loopback;
2933	if (ret) {
2934		ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n",
2935			  ret);
2936		return ret;
2937	}
2938
2939	mask = IB_QP_STATE | IB_QP_SQ_PSN | IB_QP_RETRY_CNT | IB_QP_TIMEOUT |
2940	       IB_QP_RNR_RETRY | IB_QP_MAX_QP_RD_ATOMIC;
2941	attr->qp_state = IB_QPS_RTS;
2942	attr->sq_psn = HNS_ROCE_FREE_MR_USED_PSN;
2943	attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT;
2944	attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT;
2945	ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_RTR,
2946				    IB_QPS_RTS, NULL);
2947	if (ret)
2948		ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n",
2949			  ret);
2950
2951	return ret;
2952}
2953
2954static int free_mr_modify_qp(struct hns_roce_dev *hr_dev)
2955{
2956	struct hns_roce_v2_priv *priv = hr_dev->priv;
2957	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2958	struct ib_qp_attr attr = {};
2959	int ret;
2960	int i;
2961
2962	rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0);
2963	rdma_ah_set_static_rate(&attr.ah_attr, 3);
2964	rdma_ah_set_port_num(&attr.ah_attr, 1);
2965
2966	for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
2967		ret = free_mr_modify_rsv_qp(hr_dev, &attr, i);
2968		if (ret)
2969			return ret;
2970	}
2971
2972	return 0;
2973}
2974
2975static int free_mr_init(struct hns_roce_dev *hr_dev)
2976{
2977	struct hns_roce_v2_priv *priv = hr_dev->priv;
2978	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2979	int ret;
2980
2981	mutex_init(&free_mr->mutex);
2982
2983	ret = free_mr_alloc_res(hr_dev);
2984	if (ret) {
2985		mutex_destroy(&free_mr->mutex);
2986		return ret;
2987	}
2988
2989	ret = free_mr_modify_qp(hr_dev);
2990	if (ret)
2991		goto err_modify_qp;
2992
2993	return 0;
2994
2995err_modify_qp:
2996	free_mr_exit(hr_dev);
2997
2998	return ret;
2999}
3000
3001static int get_hem_table(struct hns_roce_dev *hr_dev)
3002{
3003	unsigned int qpc_count;
3004	unsigned int cqc_count;
3005	unsigned int gmv_count;
3006	int ret;
3007	int i;
3008
3009	/* Alloc memory for source address table buffer space chunk */
3010	for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num;
3011	     gmv_count++) {
3012		ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count);
3013		if (ret)
3014			goto err_gmv_failed;
3015	}
3016
3017	if (hr_dev->is_vf)
3018		return 0;
3019
3020	/* Alloc memory for QPC Timer buffer space chunk */
3021	for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
3022	     qpc_count++) {
3023		ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
3024					 qpc_count);
3025		if (ret) {
3026			dev_err(hr_dev->dev, "QPC Timer get failed\n");
3027			goto err_qpc_timer_failed;
3028		}
3029	}
3030
3031	/* Alloc memory for CQC Timer buffer space chunk */
3032	for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
3033	     cqc_count++) {
3034		ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
3035					 cqc_count);
3036		if (ret) {
3037			dev_err(hr_dev->dev, "CQC Timer get failed\n");
3038			goto err_cqc_timer_failed;
3039		}
3040	}
3041
3042	return 0;
3043
3044err_cqc_timer_failed:
3045	for (i = 0; i < cqc_count; i++)
3046		hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
3047
3048err_qpc_timer_failed:
3049	for (i = 0; i < qpc_count; i++)
3050		hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
3051
3052err_gmv_failed:
3053	for (i = 0; i < gmv_count; i++)
3054		hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);
3055
3056	return ret;
3057}
3058
3059static void put_hem_table(struct hns_roce_dev *hr_dev)
3060{
3061	int i;
3062
3063	for (i = 0; i < hr_dev->caps.gmv_entry_num; i++)
3064		hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);
3065
3066	if (hr_dev->is_vf)
3067		return;
3068
3069	for (i = 0; i < hr_dev->caps.qpc_timer_bt_num; i++)
3070		hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
3071
3072	for (i = 0; i < hr_dev->caps.cqc_timer_bt_num; i++)
3073		hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
3074}
3075
3076static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
3077{
3078	int ret;
3079
3080	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
3081		ret = free_mr_init(hr_dev);
3082		if (ret) {
3083			dev_err(hr_dev->dev, "failed to init free mr!\n");
3084			return ret;
3085		}
3086	}
3087
3088	/* The hns ROCEE requires the extdb info to be cleared before using */
3089	ret = hns_roce_clear_extdb_list_info(hr_dev);
3090	if (ret)
3091		goto err_clear_extdb_failed;
3092
3093	ret = get_hem_table(hr_dev);
3094	if (ret)
3095		goto err_get_hem_table_failed;
3096
3097	if (hr_dev->is_vf)
3098		return 0;
3099
3100	ret = hns_roce_init_link_table(hr_dev);
3101	if (ret) {
3102		dev_err(hr_dev->dev, "failed to init llm, ret = %d.\n", ret);
3103		goto err_llm_init_failed;
3104	}
3105
3106	return 0;
3107
3108err_llm_init_failed:
3109	put_hem_table(hr_dev);
3110err_get_hem_table_failed:
3111	hns_roce_function_clear(hr_dev);
3112err_clear_extdb_failed:
3113	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
3114		free_mr_exit(hr_dev);
3115
3116	return ret;
3117}
3118
3119static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
3120{
3121	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
3122		free_mr_exit(hr_dev);
3123
3124	hns_roce_function_clear(hr_dev);
3125
3126	if (!hr_dev->is_vf)
3127		hns_roce_free_link_table(hr_dev);
3128
3129	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
3130		free_dip_entry(hr_dev);
3131}
3132
3133static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev,
3134			      struct hns_roce_mbox_msg *mbox_msg)
3135{
3136	struct hns_roce_cmq_desc desc;
3137	struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;
3138
3139	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
3140
3141	mb->in_param_l = cpu_to_le32(mbox_msg->in_param);
3142	mb->in_param_h = cpu_to_le32(mbox_msg->in_param >> 32);
3143	mb->out_param_l = cpu_to_le32(mbox_msg->out_param);
3144	mb->out_param_h = cpu_to_le32(mbox_msg->out_param >> 32);
3145	mb->cmd_tag = cpu_to_le32(mbox_msg->tag << 8 | mbox_msg->cmd);
3146	mb->token_event_en = cpu_to_le32(mbox_msg->event_en << 16 |
3147					 mbox_msg->token);
3148
3149	return hns_roce_cmq_send(hr_dev, &desc, 1);
3150}
3151
3152static int v2_wait_mbox_complete(struct hns_roce_dev *hr_dev, u32 timeout,
3153				 u8 *complete_status)
3154{
3155	struct hns_roce_mbox_status *mb_st;
3156	struct hns_roce_cmq_desc desc;
3157	unsigned long end;
3158	int ret = -EBUSY;
3159	u32 status;
3160	bool busy;
3161
3162	mb_st = (struct hns_roce_mbox_status *)desc.data;
3163	end = msecs_to_jiffies(timeout) + jiffies;
3164	while (v2_chk_mbox_is_avail(hr_dev, &busy)) {
3165		if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)
3166			return -EIO;
3167
3168		status = 0;
3169		hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST,
3170					      true);
3171		ret = __hns_roce_cmq_send(hr_dev, &desc, 1);
3172		if (!ret) {
3173			status = le32_to_cpu(mb_st->mb_status_hw_run);
3174			/* No pending message exists in ROCEE mbox. */
3175			if (!(status & MB_ST_HW_RUN_M))
3176				break;
3177		} else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) {
3178			break;
3179		}
3180
3181		if (time_after(jiffies, end)) {
3182			dev_err_ratelimited(hr_dev->dev,
3183					    "failed to wait mbox status 0x%x\n",
3184					    status);
3185			return -ETIMEDOUT;
3186		}
3187
3188		cond_resched();
3189		ret = -EBUSY;
3190	}
3191
3192	if (!ret) {
3193		*complete_status = (u8)(status & MB_ST_COMPLETE_M);
3194	} else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) {
3195		/* Ignore all errors if the mbox is unavailable. */
3196		ret = 0;
3197		*complete_status = MB_ST_COMPLETE_M;
3198	}
3199
3200	return ret;
3201}
3202
3203static int v2_post_mbox(struct hns_roce_dev *hr_dev,
3204			struct hns_roce_mbox_msg *mbox_msg)
3205{
3206	u8 status = 0;
3207	int ret;
3208
3209	/* Waiting for the mbox to be idle */
3210	ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS,
3211				    &status);
3212	if (unlikely(ret)) {
3213		dev_err_ratelimited(hr_dev->dev,
3214				    "failed to check post mbox status = 0x%x, ret = %d.\n",
3215				    status, ret);
3216		return ret;
3217	}
3218
3219	/* Post new message to mbox */
3220	ret = hns_roce_mbox_post(hr_dev, mbox_msg);
3221	if (ret)
3222		dev_err_ratelimited(hr_dev->dev,
3223				    "failed to post mailbox, ret = %d.\n", ret);
3224
3225	return ret;
3226}
3227
3228static int v2_poll_mbox_done(struct hns_roce_dev *hr_dev)
3229{
3230	u8 status = 0;
3231	int ret;
3232
3233	ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_CMD_TIMEOUT_MSECS,
3234				    &status);
3235	if (!ret) {
3236		if (status != MB_ST_COMPLETE_SUCC)
3237			return -EBUSY;
3238	} else {
3239		dev_err_ratelimited(hr_dev->dev,
3240				    "failed to check mbox status = 0x%x, ret = %d.\n",
3241				    status, ret);
3242	}
3243
3244	return ret;
3245}
3246
3247static void copy_gid(void *dest, const union ib_gid *gid)
3248{
3249#define GID_SIZE 4
3250	const union ib_gid *src = gid;
3251	__le32 (*p)[GID_SIZE] = dest;
3252	int i;
3253
3254	if (!gid)
3255		src = &zgid;
3256
3257	for (i = 0; i < GID_SIZE; i++)
3258		(*p)[i] = cpu_to_le32(*(u32 *)&src->raw[i * sizeof(u32)]);
3259}
3260
3261static int config_sgid_table(struct hns_roce_dev *hr_dev,
3262			     int gid_index, const union ib_gid *gid,
3263			     enum hns_roce_sgid_type sgid_type)
3264{
3265	struct hns_roce_cmq_desc desc;
3266	struct hns_roce_cfg_sgid_tb *sgid_tb =
3267				    (struct hns_roce_cfg_sgid_tb *)desc.data;
3268
3269	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
3270
3271	hr_reg_write(sgid_tb, CFG_SGID_TB_TABLE_IDX, gid_index);
3272	hr_reg_write(sgid_tb, CFG_SGID_TB_VF_SGID_TYPE, sgid_type);
3273
3274	copy_gid(&sgid_tb->vf_sgid_l, gid);
3275
3276	return hns_roce_cmq_send(hr_dev, &desc, 1);
3277}
3278
3279static int config_gmv_table(struct hns_roce_dev *hr_dev,
3280			    int gid_index, const union ib_gid *gid,
3281			    enum hns_roce_sgid_type sgid_type,
3282			    const struct ib_gid_attr *attr)
3283{
3284	struct hns_roce_cmq_desc desc[2];
3285	struct hns_roce_cfg_gmv_tb_a *tb_a =
3286				(struct hns_roce_cfg_gmv_tb_a *)desc[0].data;
3287	struct hns_roce_cfg_gmv_tb_b *tb_b =
3288				(struct hns_roce_cfg_gmv_tb_b *)desc[1].data;
3289
3290	u16 vlan_id = VLAN_CFI_MASK;
3291	u8 mac[ETH_ALEN] = {};
3292	int ret;
3293
3294	if (gid) {
3295		ret = rdma_read_gid_l2_fields(attr, &vlan_id, mac);
3296		if (ret)
3297			return ret;
3298	}
3299
3300	hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_CFG_GMV_TBL, false);
3301	desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
3302
3303	hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_CFG_GMV_TBL, false);
3304
3305	copy_gid(&tb_a->vf_sgid_l, gid);
3306
3307	hr_reg_write(tb_a, GMV_TB_A_VF_SGID_TYPE, sgid_type);
3308	hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_EN, vlan_id < VLAN_CFI_MASK);
3309	hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_ID, vlan_id);
3310
3311	tb_b->vf_smac_l = cpu_to_le32(*(u32 *)mac);
3312
3313	hr_reg_write(tb_b, GMV_TB_B_SMAC_H, *(u16 *)&mac[4]);
3314	hr_reg_write(tb_b, GMV_TB_B_SGID_IDX, gid_index);
3315
3316	return hns_roce_cmq_send(hr_dev, desc, 2);
3317}
3318
3319static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, int gid_index,
3320			       const union ib_gid *gid,
3321			       const struct ib_gid_attr *attr)
3322{
3323	enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
3324	int ret;
3325
3326	if (gid) {
3327		if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
3328			if (ipv6_addr_v4mapped((void *)gid))
3329				sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
3330			else
3331				sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
3332		} else if (attr->gid_type == IB_GID_TYPE_ROCE) {
3333			sgid_type = GID_TYPE_FLAG_ROCE_V1;
3334		}
3335	}
3336
3337	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
3338		ret = config_gmv_table(hr_dev, gid_index, gid, sgid_type, attr);
3339	else
3340		ret = config_sgid_table(hr_dev, gid_index, gid, sgid_type);
3341
3342	if (ret)
3343		ibdev_err(&hr_dev->ib_dev, "failed to set gid, ret = %d!\n",
3344			  ret);
3345
3346	return ret;
3347}
3348
3349static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
3350			       const u8 *addr)
3351{
3352	struct hns_roce_cmq_desc desc;
3353	struct hns_roce_cfg_smac_tb *smac_tb =
3354				    (struct hns_roce_cfg_smac_tb *)desc.data;
3355	u16 reg_smac_h;
3356	u32 reg_smac_l;
3357
3358	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);
3359
3360	reg_smac_l = *(u32 *)(&addr[0]);
3361	reg_smac_h = *(u16 *)(&addr[4]);
3362
3363	hr_reg_write(smac_tb, CFG_SMAC_TB_IDX, phy_port);
3364	hr_reg_write(smac_tb, CFG_SMAC_TB_VF_SMAC_H, reg_smac_h);
3365	smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);
3366
3367	return hns_roce_cmq_send(hr_dev, &desc, 1);
3368}
3369
3370static int set_mtpt_pbl(struct hns_roce_dev *hr_dev,
3371			struct hns_roce_v2_mpt_entry *mpt_entry,
3372			struct hns_roce_mr *mr)
3373{
3374	u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };
3375	struct ib_device *ibdev = &hr_dev->ib_dev;
3376	dma_addr_t pbl_ba;
3377	int ret;
3378	int i;
3379
3380	ret = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
3381				min_t(int, ARRAY_SIZE(pages), mr->npages));
3382	if (ret) {
3383		ibdev_err(ibdev, "failed to find PBL mtr, ret = %d.\n", ret);
3384		return ret;
3385	}
3386
3387	/* Aligned to the hardware address access unit */
3388	for (i = 0; i < ARRAY_SIZE(pages); i++)
3389		pages[i] >>= MPT_PBL_BUF_ADDR_S;
3390
3391	pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr);
3392
3393	mpt_entry->pbl_size = cpu_to_le32(mr->npages);
3394	mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> MPT_PBL_BA_ADDR_S);
3395	hr_reg_write(mpt_entry, MPT_PBL_BA_H,
3396		     upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S));
3397
3398	mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
3399	hr_reg_write(mpt_entry, MPT_PA0_H, upper_32_bits(pages[0]));
3400
3401	mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
3402	hr_reg_write(mpt_entry, MPT_PA1_H, upper_32_bits(pages[1]));
3403	hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,
3404		     to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
3405
3406	return 0;
3407}
3408
3409static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,
3410				  void *mb_buf, struct hns_roce_mr *mr)
3411{
3412	struct hns_roce_v2_mpt_entry *mpt_entry;
3413
3414	mpt_entry = mb_buf;
3415	memset(mpt_entry, 0, sizeof(*mpt_entry));
3416
3417	hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);
3418	hr_reg_write(mpt_entry, MPT_PD, mr->pd);
3419
3420	hr_reg_write_bool(mpt_entry, MPT_ATOMIC_EN,
3421			  mr->access & IB_ACCESS_REMOTE_ATOMIC);
3422	hr_reg_write_bool(mpt_entry, MPT_RR_EN,
3423			  mr->access & IB_ACCESS_REMOTE_READ);
3424	hr_reg_write_bool(mpt_entry, MPT_RW_EN,
3425			  mr->access & IB_ACCESS_REMOTE_WRITE);
3426	hr_reg_write_bool(mpt_entry, MPT_LW_EN,
3427			  mr->access & IB_ACCESS_LOCAL_WRITE);
3428
3429	mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
3430	mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
3431	mpt_entry->lkey = cpu_to_le32(mr->key);
3432	mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
3433	mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
3434
3435	if (mr->type != MR_TYPE_MR)
3436		hr_reg_enable(mpt_entry, MPT_PA);
3437
3438	if (mr->type == MR_TYPE_DMA)
3439		return 0;
3440
3441	if (mr->pbl_hop_num != HNS_ROCE_HOP_NUM_0)
3442		hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, mr->pbl_hop_num);
3443
3444	hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,
3445		     to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
3446	hr_reg_enable(mpt_entry, MPT_INNER_PA_VLD);
3447
3448	return set_mtpt_pbl(hr_dev, mpt_entry, mr);
3449}
3450
3451static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
3452					struct hns_roce_mr *mr, int flags,
3453					void *mb_buf)
3454{
3455	struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
3456	u32 mr_access_flags = mr->access;
3457	int ret = 0;
3458
3459	hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);
3460	hr_reg_write(mpt_entry, MPT_PD, mr->pd);
3461
3462	if (flags & IB_MR_REREG_ACCESS) {
3463		hr_reg_write(mpt_entry, MPT_ATOMIC_EN,
3464			     mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
3465		hr_reg_write(mpt_entry, MPT_RR_EN,
3466			     mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
3467		hr_reg_write(mpt_entry, MPT_RW_EN,
3468			     mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
3469		hr_reg_write(mpt_entry, MPT_LW_EN,
3470			     mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
3471	}
3472
3473	if (flags & IB_MR_REREG_TRANS) {
3474		mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
3475		mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
3476		mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
3477		mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
3478
3479		ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
3480	}
3481
3482	return ret;
3483}
3484
3485static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
3486{
3487	dma_addr_t pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr);
3488	struct hns_roce_v2_mpt_entry *mpt_entry;
3489
3490	mpt_entry = mb_buf;
3491	memset(mpt_entry, 0, sizeof(*mpt_entry));
3492
3493	hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE);
3494	hr_reg_write(mpt_entry, MPT_PD, mr->pd);
3495
3496	hr_reg_enable(mpt_entry, MPT_RA_EN);
3497	hr_reg_enable(mpt_entry, MPT_R_INV_EN);
3498
3499	hr_reg_enable(mpt_entry, MPT_FRE);
3500	hr_reg_enable(mpt_entry, MPT_BPD);
3501	hr_reg_clear(mpt_entry, MPT_PA);
3502
3503	hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, 1);
3504	hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,
3505		     to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
3506	hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,
3507		     to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
3508
3509	mpt_entry->pbl_size = cpu_to_le32(mr->npages);
3510
3511	mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >>
3512							MPT_PBL_BA_ADDR_S));
3513	hr_reg_write(mpt_entry, MPT_PBL_BA_H,
3514		     upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S));
3515
3516	return 0;
3517}
3518
3519static int free_mr_post_send_lp_wqe(struct hns_roce_qp *hr_qp)
3520{
3521	struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);
3522	struct ib_device *ibdev = &hr_dev->ib_dev;
3523	const struct ib_send_wr *bad_wr;
3524	struct ib_rdma_wr rdma_wr = {};
3525	struct ib_send_wr *send_wr;
3526	int ret;
3527
3528	send_wr = &rdma_wr.wr;
3529	send_wr->opcode = IB_WR_RDMA_WRITE;
3530
3531	ret = hns_roce_v2_post_send(&hr_qp->ibqp, send_wr, &bad_wr);
3532	if (ret) {
3533		ibdev_err_ratelimited(ibdev, "failed to post wqe for free mr, ret = %d.\n",
3534				      ret);
3535		return ret;
3536	}
3537
3538	return 0;
3539}
3540
3541static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
3542			       struct ib_wc *wc);
3543
3544static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev)
3545{
3546	struct hns_roce_v2_priv *priv = hr_dev->priv;
3547	struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
3548	struct ib_wc wc[ARRAY_SIZE(free_mr->rsv_qp)];
3549	struct ib_device *ibdev = &hr_dev->ib_dev;
3550	struct hns_roce_qp *hr_qp;
3551	unsigned long end;
3552	int cqe_cnt = 0;
3553	int npolled;
3554	int ret;
3555	int i;
3556
3557	/*
3558	 * If the device initialization is not complete or in the uninstall
3559	 * process, then there is no need to execute free mr.
3560	 */
3561	if (priv->handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT ||
3562	    priv->handle->rinfo.instance_state == HNS_ROCE_STATE_INIT ||
3563	    hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT)
3564		return;
3565
3566	mutex_lock(&free_mr->mutex);
3567
3568	for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
3569		hr_qp = free_mr->rsv_qp[i];
3570
3571		ret = free_mr_post_send_lp_wqe(hr_qp);
3572		if (ret) {
3573			ibdev_err_ratelimited(ibdev,
3574					      "failed to send wqe (qp:0x%lx) for free mr, ret = %d.\n",
3575					      hr_qp->qpn, ret);
3576			break;
3577		}
3578
3579		cqe_cnt++;
3580	}
3581
3582	end = msecs_to_jiffies(HNS_ROCE_V2_FREE_MR_TIMEOUT) + jiffies;
3583	while (cqe_cnt) {
3584		npolled = hns_roce_v2_poll_cq(&free_mr->rsv_cq->ib_cq, cqe_cnt, wc);
3585		if (npolled < 0) {
3586			ibdev_err_ratelimited(ibdev,
3587					      "failed to poll cqe for free mr, remain %d cqe.\n",
3588					      cqe_cnt);
3589			goto out;
3590		}
3591
3592		if (time_after(jiffies, end)) {
3593			ibdev_err_ratelimited(ibdev,
3594					      "failed to poll cqe for free mr and timeout, remain %d cqe.\n",
3595					      cqe_cnt);
3596			goto out;
3597		}
3598		cqe_cnt -= npolled;
3599	}
3600
3601out:
3602	mutex_unlock(&free_mr->mutex);
3603}
3604
3605static void hns_roce_v2_dereg_mr(struct hns_roce_dev *hr_dev)
3606{
3607	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
3608		free_mr_send_cmd_to_hw(hr_dev);
3609}
3610
3611static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
3612{
3613	return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size);
3614}
3615
3616static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, unsigned int n)
3617{
3618	struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
3619
3620	/* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
3621	return (hr_reg_read(cqe, CQE_OWNER) ^ !!(n & hr_cq->cq_depth)) ? cqe :
3622									 NULL;
3623}
3624
3625static inline void update_cq_db(struct hns_roce_dev *hr_dev,
3626				struct hns_roce_cq *hr_cq)
3627{
3628	if (likely(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)) {
3629		*hr_cq->set_ci_db = hr_cq->cons_index & V2_CQ_DB_CONS_IDX_M;
3630	} else {
3631		struct hns_roce_v2_db cq_db = {};
3632
3633		hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn);
3634		hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB);
3635		hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index);
3636		hr_reg_write(&cq_db, DB_CQ_CMD_SN, 1);
3637
3638		hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg);
3639	}
3640}
3641
3642static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
3643				   struct hns_roce_srq *srq)
3644{
3645	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
3646	struct hns_roce_v2_cqe *cqe, *dest;
3647	u32 prod_index;
3648	int nfreed = 0;
3649	int wqe_index;
3650	u8 owner_bit;
3651
3652	for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
3653	     ++prod_index) {
3654		if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)
3655			break;
3656	}
3657
3658	/*
3659	 * Now backwards through the CQ, removing CQ entries
3660	 * that match our QP by overwriting them with next entries.
3661	 */
3662	while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
3663		cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
3664		if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) {
3665			if (srq && hr_reg_read(cqe, CQE_S_R)) {
3666				wqe_index = hr_reg_read(cqe, CQE_WQE_IDX);
3667				hns_roce_free_srq_wqe(srq, wqe_index);
3668			}
3669			++nfreed;
3670		} else if (nfreed) {
3671			dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
3672					  hr_cq->ib_cq.cqe);
3673			owner_bit = hr_reg_read(dest, CQE_OWNER);
3674			memcpy(dest, cqe, hr_cq->cqe_size);
3675			hr_reg_write(dest, CQE_OWNER, owner_bit);
3676		}
3677	}
3678
3679	if (nfreed) {
3680		hr_cq->cons_index += nfreed;
3681		update_cq_db(hr_dev, hr_cq);
3682	}
3683}
3684
3685static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
3686				 struct hns_roce_srq *srq)
3687{
3688	spin_lock_irq(&hr_cq->lock);
3689	__hns_roce_v2_cq_clean(hr_cq, qpn, srq);
3690	spin_unlock_irq(&hr_cq->lock);
3691}
3692
3693static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
3694				  struct hns_roce_cq *hr_cq, void *mb_buf,
3695				  u64 *mtts, dma_addr_t dma_handle)
3696{
3697	struct hns_roce_v2_cq_context *cq_context;
3698
3699	cq_context = mb_buf;
3700	memset(cq_context, 0, sizeof(*cq_context));
3701
3702	hr_reg_write(cq_context, CQC_CQ_ST, V2_CQ_STATE_VALID);
3703	hr_reg_write(cq_context, CQC_ARM_ST, NO_ARMED);
3704	hr_reg_write(cq_context, CQC_SHIFT, ilog2(hr_cq->cq_depth));
3705	hr_reg_write(cq_context, CQC_CEQN, hr_cq->vector);
3706	hr_reg_write(cq_context, CQC_CQN, hr_cq->cqn);
3707
3708	if (hr_cq->cqe_size == HNS_ROCE_V3_CQE_SIZE)
3709		hr_reg_write(cq_context, CQC_CQE_SIZE, CQE_SIZE_64B);
3710
3711	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)
3712		hr_reg_enable(cq_context, CQC_STASH);
3713
3714	hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_L,
3715		     to_hr_hw_page_addr(mtts[0]));
3716	hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_H,
3717		     upper_32_bits(to_hr_hw_page_addr(mtts[0])));
3718	hr_reg_write(cq_context, CQC_CQE_HOP_NUM, hr_dev->caps.cqe_hop_num ==
3719		     HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
3720	hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_L,
3721		     to_hr_hw_page_addr(mtts[1]));
3722	hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_H,
3723		     upper_32_bits(to_hr_hw_page_addr(mtts[1])));
3724	hr_reg_write(cq_context, CQC_CQE_BAR_PG_SZ,
3725		     to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift));
3726	hr_reg_write(cq_context, CQC_CQE_BUF_PG_SZ,
3727		     to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift));
3728	hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle >> CQC_CQE_BA_L_S);
3729	hr_reg_write(cq_context, CQC_CQE_BA_H, dma_handle >> CQC_CQE_BA_H_S);
3730	hr_reg_write_bool(cq_context, CQC_DB_RECORD_EN,
3731			  hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB);
3732	hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_L,
3733		     ((u32)hr_cq->db.dma) >> 1);
3734	hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_H,
3735		     hr_cq->db.dma >> CQC_CQE_DB_RECORD_ADDR_H_S);
3736	hr_reg_write(cq_context, CQC_CQ_MAX_CNT,
3737		     HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
3738	hr_reg_write(cq_context, CQC_CQ_PERIOD,
3739		     HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
3740}
3741
3742static bool left_sw_wc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
3743{
3744	struct hns_roce_qp *hr_qp;
3745
3746	list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) {
3747		if (hr_qp->sq.head != hr_qp->sq.tail)
3748			return true;
3749	}
3750
3751	list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) {
3752		if (hr_qp->rq.head != hr_qp->rq.tail)
3753			return true;
3754	}
3755
3756	return false;
3757}
3758
3759static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
3760				     enum ib_cq_notify_flags flags)
3761{
3762	struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
3763	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
3764	struct hns_roce_v2_db cq_db = {};
3765	u32 notify_flag;
3766
3767	if (hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN) {
3768		if ((flags & IB_CQ_REPORT_MISSED_EVENTS) &&
3769		    left_sw_wc(hr_dev, hr_cq))
3770			return 1;
3771		return 0;
3772	}
3773	/*
3774	 * flags = 0, then notify_flag : next
3775	 * flags = 1, then notify flag : solocited
3776	 */
3777	notify_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
3778		      V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
3779
3780	hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn);
3781	hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB_NOTIFY);
3782	hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index);
3783	hr_reg_write(&cq_db, DB_CQ_CMD_SN, hr_cq->arm_sn);
3784	hr_reg_write(&cq_db, DB_CQ_NOTIFY, notify_flag);
3785
3786	hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg);
3787
3788	return 0;
3789}
3790
3791static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq,
3792		   int num_entries, struct ib_wc *wc)
3793{
3794	unsigned int left;
3795	int npolled = 0;
3796
3797	left = wq->head - wq->tail;
3798	if (left == 0)
3799		return 0;
3800
3801	left = min_t(unsigned int, (unsigned int)num_entries, left);
3802	while (npolled < left) {
3803		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3804		wc->status = IB_WC_WR_FLUSH_ERR;
3805		wc->vendor_err = 0;
3806		wc->qp = &hr_qp->ibqp;
3807
3808		wq->tail++;
3809		wc++;
3810		npolled++;
3811	}
3812
3813	return npolled;
3814}
3815
3816static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries,
3817				  struct ib_wc *wc)
3818{
3819	struct hns_roce_qp *hr_qp;
3820	int npolled = 0;
3821
3822	list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) {
3823		npolled += sw_comp(hr_qp, &hr_qp->sq,
3824				   num_entries - npolled, wc + npolled);
3825		if (npolled >= num_entries)
3826			goto out;
3827	}
3828
3829	list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) {
3830		npolled += sw_comp(hr_qp, &hr_qp->rq,
3831				   num_entries - npolled, wc + npolled);
3832		if (npolled >= num_entries)
3833			goto out;
3834	}
3835
3836out:
3837	return npolled;
3838}
3839
3840static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,
3841			   struct hns_roce_cq *cq, struct hns_roce_v2_cqe *cqe,
3842			   struct ib_wc *wc)
3843{
3844	static const struct {
3845		u32 cqe_status;
3846		enum ib_wc_status wc_status;
3847	} map[] = {
3848		{ HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS },
3849		{ HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR },
3850		{ HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },
3851		{ HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR },
3852		{ HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },
3853		{ HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR },
3854		{ HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },
3855		{ HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },
3856		{ HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR },
3857		{ HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },
3858		{ HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR },
3859		{ HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR,
3860		  IB_WC_RETRY_EXC_ERR },
3861		{ HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },
3862		{ HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },
3863		{ HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR}
3864	};
3865
3866	u32 cqe_status = hr_reg_read(cqe, CQE_STATUS);
3867	int i;
3868
3869	wc->status = IB_WC_GENERAL_ERR;
3870	for (i = 0; i < ARRAY_SIZE(map); i++)
3871		if (cqe_status == map[i].cqe_status) {
3872			wc->status = map[i].wc_status;
3873			break;
3874		}
3875
3876	if (likely(wc->status == IB_WC_SUCCESS ||
3877		   wc->status == IB_WC_WR_FLUSH_ERR))
3878		return;
3879
3880	ibdev_err_ratelimited(&hr_dev->ib_dev, "error cqe status 0x%x:\n",
3881			      cqe_status);
3882	print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_NONE, 16, 4, cqe,
3883		       cq->cqe_size, false);
3884	wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS);
3885
3886	/*
3887	 * For hns ROCEE, GENERAL_ERR is an error type that is not defined in
3888	 * the standard protocol, the driver must ignore it and needn't to set
3889	 * the QP to an error state.
3890	 */
3891	if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR)
3892		return;
3893
3894	flush_cqe(hr_dev, qp);
3895}
3896
3897static int get_cur_qp(struct hns_roce_cq *hr_cq, struct hns_roce_v2_cqe *cqe,
3898		      struct hns_roce_qp **cur_qp)
3899{
3900	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
3901	struct hns_roce_qp *hr_qp = *cur_qp;
3902	u32 qpn;
3903
3904	qpn = hr_reg_read(cqe, CQE_LCL_QPN);
3905
3906	if (!hr_qp || qpn != hr_qp->qpn) {
3907		hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
3908		if (unlikely(!hr_qp)) {
3909			ibdev_err(&hr_dev->ib_dev,
3910				  "CQ %06lx with entry for unknown QPN %06x\n",
3911				  hr_cq->cqn, qpn);
3912			return -EINVAL;
3913		}
3914		*cur_qp = hr_qp;
3915	}
3916
3917	return 0;
3918}
3919
3920/*
3921 * mapped-value = 1 + real-value
3922 * The ib wc opcode's real value is start from 0, In order to distinguish
3923 * between initialized and uninitialized map values, we plus 1 to the actual
3924 * value when defining the mapping, so that the validity can be identified by
3925 * checking whether the mapped value is greater than 0.
3926 */
3927#define HR_WC_OP_MAP(hr_key, ib_key) \
3928		[HNS_ROCE_V2_WQE_OP_ ## hr_key] = 1 + IB_WC_ ## ib_key
3929
3930static const u32 wc_send_op_map[] = {
3931	HR_WC_OP_MAP(SEND,			SEND),
3932	HR_WC_OP_MAP(SEND_WITH_INV,		SEND),
3933	HR_WC_OP_MAP(SEND_WITH_IMM,		SEND),
3934	HR_WC_OP_MAP(RDMA_READ,			RDMA_READ),
3935	HR_WC_OP_MAP(RDMA_WRITE,		RDMA_WRITE),
3936	HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM,	RDMA_WRITE),
3937	HR_WC_OP_MAP(ATOM_CMP_AND_SWAP,		COMP_SWAP),
3938	HR_WC_OP_MAP(ATOM_FETCH_AND_ADD,	FETCH_ADD),
3939	HR_WC_OP_MAP(ATOM_MSK_CMP_AND_SWAP,	MASKED_COMP_SWAP),
3940	HR_WC_OP_MAP(ATOM_MSK_FETCH_AND_ADD,	MASKED_FETCH_ADD),
3941	HR_WC_OP_MAP(FAST_REG_PMR,		REG_MR),
3942};
3943
3944static int to_ib_wc_send_op(u32 hr_opcode)
3945{
3946	if (hr_opcode >= ARRAY_SIZE(wc_send_op_map))
3947		return -EINVAL;
3948
3949	return wc_send_op_map[hr_opcode] ? wc_send_op_map[hr_opcode] - 1 :
3950					   -EINVAL;
3951}
3952
3953static const u32 wc_recv_op_map[] = {
3954	HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM,		WITH_IMM),
3955	HR_WC_OP_MAP(SEND,				RECV),
3956	HR_WC_OP_MAP(SEND_WITH_IMM,			WITH_IMM),
3957	HR_WC_OP_MAP(SEND_WITH_INV,			RECV),
3958};
3959
3960static int to_ib_wc_recv_op(u32 hr_opcode)
3961{
3962	if (hr_opcode >= ARRAY_SIZE(wc_recv_op_map))
3963		return -EINVAL;
3964
3965	return wc_recv_op_map[hr_opcode] ? wc_recv_op_map[hr_opcode] - 1 :
3966					   -EINVAL;
3967}
3968
3969static void fill_send_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)
3970{
3971	u32 hr_opcode;
3972	int ib_opcode;
3973
3974	wc->wc_flags = 0;
3975
3976	hr_opcode = hr_reg_read(cqe, CQE_OPCODE);
3977	switch (hr_opcode) {
3978	case HNS_ROCE_V2_WQE_OP_RDMA_READ:
3979		wc->byte_len = le32_to_cpu(cqe->byte_cnt);
3980		break;
3981	case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM:
3982	case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM:
3983		wc->wc_flags |= IB_WC_WITH_IMM;
3984		break;
3985	case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP:
3986	case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD:
3987	case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP:
3988	case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD:
3989		wc->byte_len  = 8;
3990		break;
3991	default:
3992		break;
3993	}
3994
3995	ib_opcode = to_ib_wc_send_op(hr_opcode);
3996	if (ib_opcode < 0)
3997		wc->status = IB_WC_GENERAL_ERR;
3998	else
3999		wc->opcode = ib_opcode;
4000}
4001
4002static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)
4003{
4004	u32 hr_opcode;
4005	int ib_opcode;
4006
4007	wc->byte_len = le32_to_cpu(cqe->byte_cnt);
4008
4009	hr_opcode = hr_reg_read(cqe, CQE_OPCODE);
4010	switch (hr_opcode) {
4011	case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
4012	case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
4013		wc->wc_flags = IB_WC_WITH_IMM;
4014		wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->immtdata));
4015		break;
4016	case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
4017		wc->wc_flags = IB_WC_WITH_INVALIDATE;
4018		wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
4019		break;
4020	default:
4021		wc->wc_flags = 0;
4022	}
4023
4024	ib_opcode = to_ib_wc_recv_op(hr_opcode);
4025	if (ib_opcode < 0)
4026		wc->status = IB_WC_GENERAL_ERR;
4027	else
4028		wc->opcode = ib_opcode;
4029
4030	wc->sl = hr_reg_read(cqe, CQE_SL);
4031	wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN);
4032	wc->slid = 0;
4033	wc->wc_flags |= hr_reg_read(cqe, CQE_GRH) ? IB_WC_GRH : 0;
4034	wc->port_num = hr_reg_read(cqe, CQE_PORTN);
4035	wc->pkey_index = 0;
4036
4037	if (hr_reg_read(cqe, CQE_VID_VLD)) {
4038		wc->vlan_id = hr_reg_read(cqe, CQE_VID);
4039		wc->wc_flags |= IB_WC_WITH_VLAN;
4040	} else {
4041		wc->vlan_id = 0xffff;
4042	}
4043
4044	wc->network_hdr_type = hr_reg_read(cqe, CQE_PORT_TYPE);
4045
4046	return 0;
4047}
4048
4049static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
4050				struct hns_roce_qp **cur_qp, struct ib_wc *wc)
4051{
4052	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
4053	struct hns_roce_qp *qp = *cur_qp;
4054	struct hns_roce_srq *srq = NULL;
4055	struct hns_roce_v2_cqe *cqe;
4056	struct hns_roce_wq *wq;
4057	int is_send;
4058	u16 wqe_idx;
4059	int ret;
4060
4061	cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
4062	if (!cqe)
4063		return -EAGAIN;
4064
4065	++hr_cq->cons_index;
4066	/* Memory barrier */
4067	rmb();
4068
4069	ret = get_cur_qp(hr_cq, cqe, &qp);
4070	if (ret)
4071		return ret;
4072
4073	wc->qp = &qp->ibqp;
4074	wc->vendor_err = 0;
4075
4076	wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX);
4077
4078	is_send = !hr_reg_read(cqe, CQE_S_R);
4079	if (is_send) {
4080		wq = &qp->sq;
4081
4082		/* If sg_signal_bit is set, tail pointer will be updated to
4083		 * the WQE corresponding to the current CQE.
4084		 */
4085		if (qp->sq_signal_bits)
4086			wq->tail += (wqe_idx - (u16)wq->tail) &
4087				    (wq->wqe_cnt - 1);
4088
4089		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
4090		++wq->tail;
4091
4092		fill_send_wc(wc, cqe);
4093	} else {
4094		if (qp->ibqp.srq) {
4095			srq = to_hr_srq(qp->ibqp.srq);
4096			wc->wr_id = srq->wrid[wqe_idx];
4097			hns_roce_free_srq_wqe(srq, wqe_idx);
4098		} else {
4099			wq = &qp->rq;
4100			wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
4101			++wq->tail;
4102		}
4103
4104		ret = fill_recv_wc(wc, cqe);
4105	}
4106
4107	get_cqe_status(hr_dev, qp, hr_cq, cqe, wc);
4108	if (unlikely(wc->status != IB_WC_SUCCESS))
4109		return 0;
4110
4111	return ret;
4112}
4113
4114static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
4115			       struct ib_wc *wc)
4116{
4117	struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
4118	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
4119	struct hns_roce_qp *cur_qp = NULL;
4120	unsigned long flags;
4121	int npolled;
4122
4123	spin_lock_irqsave(&hr_cq->lock, flags);
4124
4125	/*
4126	 * When the device starts to reset, the state is RST_DOWN. At this time,
4127	 * there may still be some valid CQEs in the hardware that are not
4128	 * polled. Therefore, it is not allowed to switch to the software mode
4129	 * immediately. When the state changes to UNINIT, CQE no longer exists
4130	 * in the hardware, and then switch to software mode.
4131	 */
4132	if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) {
4133		npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc);
4134		goto out;
4135	}
4136
4137	for (npolled = 0; npolled < num_entries; ++npolled) {
4138		if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
4139			break;
4140	}
4141
4142	if (npolled)
4143		update_cq_db(hr_dev, hr_cq);
4144
4145out:
4146	spin_unlock_irqrestore(&hr_cq->lock, flags);
4147
4148	return npolled;
4149}
4150
4151static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
4152			      u32 step_idx, u8 *mbox_cmd)
4153{
4154	u8 cmd;
4155
4156	switch (type) {
4157	case HEM_TYPE_QPC:
4158		cmd = HNS_ROCE_CMD_WRITE_QPC_BT0;
4159		break;
4160	case HEM_TYPE_MTPT:
4161		cmd = HNS_ROCE_CMD_WRITE_MPT_BT0;
4162		break;
4163	case HEM_TYPE_CQC:
4164		cmd = HNS_ROCE_CMD_WRITE_CQC_BT0;
4165		break;
4166	case HEM_TYPE_SRQC:
4167		cmd = HNS_ROCE_CMD_WRITE_SRQC_BT0;
4168		break;
4169	case HEM_TYPE_SCCC:
4170		cmd = HNS_ROCE_CMD_WRITE_SCCC_BT0;
4171		break;
4172	case HEM_TYPE_QPC_TIMER:
4173		cmd = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
4174		break;
4175	case HEM_TYPE_CQC_TIMER:
4176		cmd = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
4177		break;
4178	default:
4179		dev_warn(hr_dev->dev, "failed to check hem type %u.\n", type);
4180		return -EINVAL;
4181	}
4182
4183	*mbox_cmd = cmd + step_idx;
4184
4185	return 0;
4186}
4187
4188static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,
4189			       dma_addr_t base_addr)
4190{
4191	struct hns_roce_cmq_desc desc;
4192	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
4193	u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz);
4194	u64 addr = to_hr_hw_page_addr(base_addr);
4195
4196	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false);
4197
4198	hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));
4199	hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));
4200	hr_reg_write(req, CFG_GMV_BT_IDX, idx);
4201
4202	return hns_roce_cmq_send(hr_dev, &desc, 1);
4203}
4204
4205static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj,
4206			 dma_addr_t base_addr, u32 hem_type, u32 step_idx)
4207{
4208	int ret;
4209	u8 cmd;
4210
4211	if (unlikely(hem_type == HEM_TYPE_GMV))
4212		return config_gmv_ba_to_hw(hr_dev, obj, base_addr);
4213
4214	if (unlikely(hem_type == HEM_TYPE_SCCC && step_idx))
4215		return 0;
4216
4217	ret = get_op_for_set_hem(hr_dev, hem_type, step_idx, &cmd);
4218	if (ret < 0)
4219		return ret;
4220
4221	return config_hem_ba_to_hw(hr_dev, base_addr, cmd, obj);
4222}
4223
4224static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
4225			       struct hns_roce_hem_table *table, int obj,
4226			       u32 step_idx)
4227{
4228	struct hns_roce_hem_mhop mhop;
4229	struct hns_roce_hem *hem;
4230	unsigned long mhop_obj = obj;
4231	int i, j, k;
4232	int ret = 0;
4233	u64 hem_idx = 0;
4234	u64 l1_idx = 0;
4235	u64 bt_ba = 0;
4236	u32 chunk_ba_num;
4237	u32 hop_num;
4238
4239	if (!hns_roce_check_whether_mhop(hr_dev, table->type))
4240		return 0;
4241
4242	hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
4243	i = mhop.l0_idx;
4244	j = mhop.l1_idx;
4245	k = mhop.l2_idx;
4246	hop_num = mhop.hop_num;
4247	chunk_ba_num = mhop.bt_chunk_size / 8;
4248
4249	if (hop_num == 2) {
4250		hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
4251			  k;
4252		l1_idx = i * chunk_ba_num + j;
4253	} else if (hop_num == 1) {
4254		hem_idx = i * chunk_ba_num + j;
4255	} else if (hop_num == HNS_ROCE_HOP_NUM_0) {
4256		hem_idx = i;
4257	}
4258
4259	if (table->type == HEM_TYPE_SCCC)
4260		obj = mhop.l0_idx;
4261
4262	if (check_whether_last_step(hop_num, step_idx)) {
4263		hem = table->hem[hem_idx];
4264
4265		ret = set_hem_to_hw(hr_dev, obj, hem->dma, table->type, step_idx);
4266	} else {
4267		if (step_idx == 0)
4268			bt_ba = table->bt_l0_dma_addr[i];
4269		else if (step_idx == 1 && hop_num == 2)
4270			bt_ba = table->bt_l1_dma_addr[l1_idx];
4271
4272		ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx);
4273	}
4274
4275	return ret;
4276}
4277
4278static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
4279				 struct hns_roce_hem_table *table,
4280				 int tag, u32 step_idx)
4281{
4282	struct hns_roce_cmd_mailbox *mailbox;
4283	struct device *dev = hr_dev->dev;
4284	u8 cmd = 0xff;
4285	int ret;
4286
4287	if (!hns_roce_check_whether_mhop(hr_dev, table->type))
4288		return 0;
4289
4290	switch (table->type) {
4291	case HEM_TYPE_QPC:
4292		cmd = HNS_ROCE_CMD_DESTROY_QPC_BT0;
4293		break;
4294	case HEM_TYPE_MTPT:
4295		cmd = HNS_ROCE_CMD_DESTROY_MPT_BT0;
4296		break;
4297	case HEM_TYPE_CQC:
4298		cmd = HNS_ROCE_CMD_DESTROY_CQC_BT0;
4299		break;
4300	case HEM_TYPE_SRQC:
4301		cmd = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
4302		break;
4303	case HEM_TYPE_SCCC:
4304	case HEM_TYPE_QPC_TIMER:
4305	case HEM_TYPE_CQC_TIMER:
4306	case HEM_TYPE_GMV:
4307		return 0;
4308	default:
4309		dev_warn(dev, "table %u not to be destroyed by mailbox!\n",
4310			 table->type);
4311		return 0;
4312	}
4313
4314	cmd += step_idx;
4315
4316	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4317	if (IS_ERR(mailbox))
4318		return PTR_ERR(mailbox);
4319
4320	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cmd, tag);
4321
4322	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4323	return ret;
4324}
4325
4326static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
4327				 struct hns_roce_v2_qp_context *context,
4328				 struct hns_roce_v2_qp_context *qpc_mask,
4329				 struct hns_roce_qp *hr_qp)
4330{
4331	struct hns_roce_cmd_mailbox *mailbox;
4332	int qpc_size;
4333	int ret;
4334
4335	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4336	if (IS_ERR(mailbox))
4337		return PTR_ERR(mailbox);
4338
4339	/* The qpc size of HIP08 is only 256B, which is half of HIP09 */
4340	qpc_size = hr_dev->caps.qpc_sz;
4341	memcpy(mailbox->buf, context, qpc_size);
4342	memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size);
4343
4344	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,
4345				HNS_ROCE_CMD_MODIFY_QPC, hr_qp->qpn);
4346
4347	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4348
4349	return ret;
4350}
4351
4352static void set_access_flags(struct hns_roce_qp *hr_qp,
4353			     struct hns_roce_v2_qp_context *context,
4354			     struct hns_roce_v2_qp_context *qpc_mask,
4355			     const struct ib_qp_attr *attr, int attr_mask)
4356{
4357	u8 dest_rd_atomic;
4358	u32 access_flags;
4359
4360	dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
4361			 attr->max_dest_rd_atomic : hr_qp->resp_depth;
4362
4363	access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
4364		       attr->qp_access_flags : hr_qp->atomic_rd_en;
4365
4366	if (!dest_rd_atomic)
4367		access_flags &= IB_ACCESS_REMOTE_WRITE;
4368
4369	hr_reg_write_bool(context, QPC_RRE,
4370			  access_flags & IB_ACCESS_REMOTE_READ);
4371	hr_reg_clear(qpc_mask, QPC_RRE);
4372
4373	hr_reg_write_bool(context, QPC_RWE,
4374			  access_flags & IB_ACCESS_REMOTE_WRITE);
4375	hr_reg_clear(qpc_mask, QPC_RWE);
4376
4377	hr_reg_write_bool(context, QPC_ATE,
4378			  access_flags & IB_ACCESS_REMOTE_ATOMIC);
4379	hr_reg_clear(qpc_mask, QPC_ATE);
4380	hr_reg_write_bool(context, QPC_EXT_ATE,
4381			  access_flags & IB_ACCESS_REMOTE_ATOMIC);
4382	hr_reg_clear(qpc_mask, QPC_EXT_ATE);
4383}
4384
4385static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,
4386			    struct hns_roce_v2_qp_context *context)
4387{
4388	hr_reg_write(context, QPC_SGE_SHIFT,
4389		     to_hr_hem_entries_shift(hr_qp->sge.sge_cnt,
4390					     hr_qp->sge.sge_shift));
4391
4392	hr_reg_write(context, QPC_SQ_SHIFT, ilog2(hr_qp->sq.wqe_cnt));
4393
4394	hr_reg_write(context, QPC_RQ_SHIFT, ilog2(hr_qp->rq.wqe_cnt));
4395}
4396
4397static inline int get_cqn(struct ib_cq *ib_cq)
4398{
4399	return ib_cq ? to_hr_cq(ib_cq)->cqn : 0;
4400}
4401
4402static inline int get_pdn(struct ib_pd *ib_pd)
4403{
4404	return ib_pd ? to_hr_pd(ib_pd)->pdn : 0;
4405}
4406
4407static void modify_qp_reset_to_init(struct ib_qp *ibqp,
4408				    struct hns_roce_v2_qp_context *context)
4409{
4410	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4411	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4412
4413	/*
4414	 * In v2 engine, software pass context and context mask to hardware
4415	 * when modifying qp. If software need modify some fields in context,
4416	 * we should set all bits of the relevant fields in context mask to
4417	 * 0 at the same time, else set them to 0x1.
4418	 */
4419	hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type));
4420
4421	hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd));
4422
4423	hr_reg_write(context, QPC_RQWS, ilog2(hr_qp->rq.max_gs));
4424
4425	set_qpc_wqe_cnt(hr_qp, context);
4426
4427	/* No VLAN need to set 0xFFF */
4428	hr_reg_write(context, QPC_VLAN_ID, 0xfff);
4429
4430	if (ibqp->qp_type == IB_QPT_XRC_TGT) {
4431		context->qkey_xrcd = cpu_to_le32(hr_qp->xrcdn);
4432
4433		hr_reg_enable(context, QPC_XRC_QP_TYPE);
4434	}
4435
4436	if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
4437		hr_reg_enable(context, QPC_RQ_RECORD_EN);
4438
4439	if (hr_qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
4440		hr_reg_enable(context, QPC_OWNER_MODE);
4441
4442	hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_L,
4443		     lower_32_bits(hr_qp->rdb.dma) >> 1);
4444	hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_H,
4445		     upper_32_bits(hr_qp->rdb.dma));
4446
4447	hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq));
4448
4449	if (ibqp->srq) {
4450		hr_reg_enable(context, QPC_SRQ_EN);
4451		hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn);
4452	}
4453
4454	hr_reg_enable(context, QPC_FRE);
4455
4456	hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq));
4457
4458	if (hr_dev->caps.qpc_sz < HNS_ROCE_V3_QPC_SZ)
4459		return;
4460
4461	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)
4462		hr_reg_enable(&context->ext, QPCEX_STASH);
4463}
4464
4465static void modify_qp_init_to_init(struct ib_qp *ibqp,
4466				   struct hns_roce_v2_qp_context *context,
4467				   struct hns_roce_v2_qp_context *qpc_mask)
4468{
4469	/*
4470	 * In v2 engine, software pass context and context mask to hardware
4471	 * when modifying qp. If software need modify some fields in context,
4472	 * we should set all bits of the relevant fields in context mask to
4473	 * 0 at the same time, else set them to 0x1.
4474	 */
4475	hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type));
4476	hr_reg_clear(qpc_mask, QPC_TST);
4477
4478	hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd));
4479	hr_reg_clear(qpc_mask, QPC_PD);
4480
4481	hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq));
4482	hr_reg_clear(qpc_mask, QPC_RX_CQN);
4483
4484	hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq));
4485	hr_reg_clear(qpc_mask, QPC_TX_CQN);
4486
4487	if (ibqp->srq) {
4488		hr_reg_enable(context, QPC_SRQ_EN);
4489		hr_reg_clear(qpc_mask, QPC_SRQ_EN);
4490		hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn);
4491		hr_reg_clear(qpc_mask, QPC_SRQN);
4492	}
4493}
4494
4495static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,
4496			    struct hns_roce_qp *hr_qp,
4497			    struct hns_roce_v2_qp_context *context,
4498			    struct hns_roce_v2_qp_context *qpc_mask)
4499{
4500	u64 mtts[MTT_MIN_COUNT] = { 0 };
4501	u64 wqe_sge_ba;
4502	int ret;
4503
4504	/* Search qp buf's mtts */
4505	ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,
4506				MTT_MIN_COUNT);
4507	if (hr_qp->rq.wqe_cnt && ret) {
4508		ibdev_err(&hr_dev->ib_dev,
4509			  "failed to find QP(0x%lx) RQ WQE buf, ret = %d.\n",
4510			  hr_qp->qpn, ret);
4511		return ret;
4512	}
4513
4514	wqe_sge_ba = hns_roce_get_mtr_ba(&hr_qp->mtr);
4515
4516	context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
4517	qpc_mask->wqe_sge_ba = 0;
4518
4519	/*
4520	 * In v2 engine, software pass context and context mask to hardware
4521	 * when modifying qp. If software need modify some fields in context,
4522	 * we should set all bits of the relevant fields in context mask to
4523	 * 0 at the same time, else set them to 0x1.
4524	 */
4525	hr_reg_write(context, QPC_WQE_SGE_BA_H, wqe_sge_ba >> (32 + 3));
4526	hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_H);
4527
4528	hr_reg_write(context, QPC_SQ_HOP_NUM,
4529		     to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num,
4530				      hr_qp->sq.wqe_cnt));
4531	hr_reg_clear(qpc_mask, QPC_SQ_HOP_NUM);
4532
4533	hr_reg_write(context, QPC_SGE_HOP_NUM,
4534		     to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num,
4535				      hr_qp->sge.sge_cnt));
4536	hr_reg_clear(qpc_mask, QPC_SGE_HOP_NUM);
4537
4538	hr_reg_write(context, QPC_RQ_HOP_NUM,
4539		     to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num,
4540				      hr_qp->rq.wqe_cnt));
4541
4542	hr_reg_clear(qpc_mask, QPC_RQ_HOP_NUM);
4543
4544	hr_reg_write(context, QPC_WQE_SGE_BA_PG_SZ,
4545		     to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift));
4546	hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_PG_SZ);
4547
4548	hr_reg_write(context, QPC_WQE_SGE_BUF_PG_SZ,
4549		     to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift));
4550	hr_reg_clear(qpc_mask, QPC_WQE_SGE_BUF_PG_SZ);
4551
4552	context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
4553	qpc_mask->rq_cur_blk_addr = 0;
4554
4555	hr_reg_write(context, QPC_RQ_CUR_BLK_ADDR_H,
4556		     upper_32_bits(to_hr_hw_page_addr(mtts[0])));
4557	hr_reg_clear(qpc_mask, QPC_RQ_CUR_BLK_ADDR_H);
4558
4559	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
4560		context->rq_nxt_blk_addr =
4561				cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
4562		qpc_mask->rq_nxt_blk_addr = 0;
4563		hr_reg_write(context, QPC_RQ_NXT_BLK_ADDR_H,
4564			     upper_32_bits(to_hr_hw_page_addr(mtts[1])));
4565		hr_reg_clear(qpc_mask, QPC_RQ_NXT_BLK_ADDR_H);
4566	}
4567
4568	return 0;
4569}
4570
4571static int config_qp_sq_buf(struct hns_roce_dev *hr_dev,
4572			    struct hns_roce_qp *hr_qp,
4573			    struct hns_roce_v2_qp_context *context,
4574			    struct hns_roce_v2_qp_context *qpc_mask)
4575{
4576	struct ib_device *ibdev = &hr_dev->ib_dev;
4577	u64 sge_cur_blk = 0;
4578	u64 sq_cur_blk = 0;
4579	int ret;
4580
4581	/* search qp buf's mtts */
4582	ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->sq.offset,
4583				&sq_cur_blk, 1);
4584	if (ret) {
4585		ibdev_err(ibdev, "failed to find QP(0x%lx) SQ WQE buf, ret = %d.\n",
4586			  hr_qp->qpn, ret);
4587		return ret;
4588	}
4589	if (hr_qp->sge.sge_cnt > 0) {
4590		ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
4591					hr_qp->sge.offset, &sge_cur_blk, 1);
4592		if (ret) {
4593			ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf, ret = %d.\n",
4594				  hr_qp->qpn, ret);
4595			return ret;
4596		}
4597	}
4598
4599	/*
4600	 * In v2 engine, software pass context and context mask to hardware
4601	 * when modifying qp. If software need modify some fields in context,
4602	 * we should set all bits of the relevant fields in context mask to
4603	 * 0 at the same time, else set them to 0x1.
4604	 */
4605	hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_L,
4606		     lower_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4607	hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_H,
4608		     upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4609	hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_L);
4610	hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_H);
4611
4612	hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_L,
4613		     lower_32_bits(to_hr_hw_page_addr(sge_cur_blk)));
4614	hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_H,
4615		     upper_32_bits(to_hr_hw_page_addr(sge_cur_blk)));
4616	hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_L);
4617	hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_H);
4618
4619	hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_L,
4620		     lower_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4621	hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_H,
4622		     upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4623	hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_L);
4624	hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_H);
4625
4626	return 0;
4627}
4628
4629static inline enum ib_mtu get_mtu(struct ib_qp *ibqp,
4630				  const struct ib_qp_attr *attr)
4631{
4632	if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
4633		return IB_MTU_4096;
4634
4635	return attr->path_mtu;
4636}
4637
4638static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
4639				 const struct ib_qp_attr *attr, int attr_mask,
4640				 struct hns_roce_v2_qp_context *context,
4641				 struct hns_roce_v2_qp_context *qpc_mask,
4642				 struct ib_udata *udata)
4643{
4644	struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,
4645					  struct hns_roce_ucontext, ibucontext);
4646	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4647	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4648	struct ib_device *ibdev = &hr_dev->ib_dev;
4649	dma_addr_t trrl_ba;
4650	dma_addr_t irrl_ba;
4651	enum ib_mtu ib_mtu;
4652	u8 ack_req_freq;
4653	const u8 *smac;
4654	int lp_msg_len;
4655	u8 lp_pktn_ini;
4656	u64 *mtts;
4657	u8 *dmac;
4658	u32 port;
4659	int mtu;
4660	int ret;
4661
4662	ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask);
4663	if (ret) {
4664		ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret);
4665		return ret;
4666	}
4667
4668	/* Search IRRL's mtts */
4669	mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
4670				   hr_qp->qpn, &irrl_ba);
4671	if (!mtts) {
4672		ibdev_err(ibdev, "failed to find qp irrl_table.\n");
4673		return -EINVAL;
4674	}
4675
4676	/* Search TRRL's mtts */
4677	mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
4678				   hr_qp->qpn, &trrl_ba);
4679	if (!mtts) {
4680		ibdev_err(ibdev, "failed to find qp trrl_table.\n");
4681		return -EINVAL;
4682	}
4683
4684	if (attr_mask & IB_QP_ALT_PATH) {
4685		ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n",
4686			  attr_mask);
4687		return -EINVAL;
4688	}
4689
4690	hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba >> QPC_TRRL_BA_L_S);
4691	hr_reg_clear(qpc_mask, QPC_TRRL_BA_L);
4692	context->trrl_ba = cpu_to_le32(trrl_ba >> QPC_TRRL_BA_M_S);
4693	qpc_mask->trrl_ba = 0;
4694	hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba >> QPC_TRRL_BA_H_S);
4695	hr_reg_clear(qpc_mask, QPC_TRRL_BA_H);
4696
4697	context->irrl_ba = cpu_to_le32(irrl_ba >> QPC_IRRL_BA_L_S);
4698	qpc_mask->irrl_ba = 0;
4699	hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba >> QPC_IRRL_BA_H_S);
4700	hr_reg_clear(qpc_mask, QPC_IRRL_BA_H);
4701
4702	hr_reg_enable(context, QPC_RMT_E2E);
4703	hr_reg_clear(qpc_mask, QPC_RMT_E2E);
4704
4705	hr_reg_write(context, QPC_SIG_TYPE, hr_qp->sq_signal_bits);
4706	hr_reg_clear(qpc_mask, QPC_SIG_TYPE);
4707
4708	port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
4709
4710	smac = (const u8 *)hr_dev->dev_addr[port];
4711	dmac = (u8 *)attr->ah_attr.roce.dmac;
4712	/* when dmac equals smac or loop_idc is 1, it should loopback */
4713	if (ether_addr_equal_unaligned(dmac, smac) ||
4714	    hr_dev->loop_idc == 0x1) {
4715		hr_reg_write(context, QPC_LBI, hr_dev->loop_idc);
4716		hr_reg_clear(qpc_mask, QPC_LBI);
4717	}
4718
4719	if (attr_mask & IB_QP_DEST_QPN) {
4720		hr_reg_write(context, QPC_DQPN, attr->dest_qp_num);
4721		hr_reg_clear(qpc_mask, QPC_DQPN);
4722	}
4723
4724	memcpy(&context->dmac, dmac, sizeof(u32));
4725	hr_reg_write(context, QPC_DMAC_H, *((u16 *)(&dmac[4])));
4726	qpc_mask->dmac = 0;
4727	hr_reg_clear(qpc_mask, QPC_DMAC_H);
4728
4729	ib_mtu = get_mtu(ibqp, attr);
4730	hr_qp->path_mtu = ib_mtu;
4731
4732	mtu = ib_mtu_enum_to_int(ib_mtu);
4733	if (WARN_ON(mtu <= 0))
4734		return -EINVAL;
4735#define MIN_LP_MSG_LEN 1024
4736	/* mtu * (2 ^ lp_pktn_ini) should be in the range of 1024 to mtu */
4737	lp_msg_len = max(mtu, MIN_LP_MSG_LEN);
4738	lp_pktn_ini = ilog2(lp_msg_len / mtu);
4739
4740	if (attr_mask & IB_QP_PATH_MTU) {
4741		hr_reg_write(context, QPC_MTU, ib_mtu);
4742		hr_reg_clear(qpc_mask, QPC_MTU);
4743	}
4744
4745	hr_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini);
4746	hr_reg_clear(qpc_mask, QPC_LP_PKTN_INI);
4747
4748	/*
4749	 * There are several constraints for ACK_REQ_FREQ:
4750	 * 1. mtu * (2 ^ ACK_REQ_FREQ) should not be too large, otherwise
4751	 *    it may cause some unexpected retries when sending large
4752	 *    payload.
4753	 * 2. ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI.
4754	 * 3. ACK_REQ_FREQ must be equal to LP_PKTN_INI when using LDCP
4755	 *    or HC3 congestion control algorithm.
4756	 */
4757	if (hr_qp->cong_type == CONG_TYPE_LDCP ||
4758	    hr_qp->cong_type == CONG_TYPE_HC3 ||
4759	    hr_dev->caps.max_ack_req_msg_len < lp_msg_len)
4760		ack_req_freq = lp_pktn_ini;
4761	else
4762		ack_req_freq = ilog2(hr_dev->caps.max_ack_req_msg_len / mtu);
4763	hr_reg_write(context, QPC_ACK_REQ_FREQ, ack_req_freq);
4764	hr_reg_clear(qpc_mask, QPC_ACK_REQ_FREQ);
4765
4766	hr_reg_clear(qpc_mask, QPC_RX_REQ_PSN_ERR);
4767	hr_reg_clear(qpc_mask, QPC_RX_REQ_MSN);
4768	hr_reg_clear(qpc_mask, QPC_RX_REQ_LAST_OPTYPE);
4769
4770	context->rq_rnr_timer = 0;
4771	qpc_mask->rq_rnr_timer = 0;
4772
4773	hr_reg_clear(qpc_mask, QPC_TRRL_HEAD_MAX);
4774	hr_reg_clear(qpc_mask, QPC_TRRL_TAIL_MAX);
4775
4776#define MAX_LP_SGEN 3
4777	/* rocee send 2^lp_sgen_ini segs every time */
4778	hr_reg_write(context, QPC_LP_SGEN_INI, MAX_LP_SGEN);
4779	hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI);
4780
4781	if (udata && ibqp->qp_type == IB_QPT_RC &&
4782	    (uctx->config & HNS_ROCE_RQ_INLINE_FLAGS)) {
4783		hr_reg_write_bool(context, QPC_RQIE,
4784				  hr_dev->caps.flags &
4785				  HNS_ROCE_CAP_FLAG_RQ_INLINE);
4786		hr_reg_clear(qpc_mask, QPC_RQIE);
4787	}
4788
4789	if (udata &&
4790	    (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_XRC_TGT) &&
4791	    (uctx->config & HNS_ROCE_CQE_INLINE_FLAGS)) {
4792		hr_reg_write_bool(context, QPC_CQEIE,
4793				  hr_dev->caps.flags &
4794				  HNS_ROCE_CAP_FLAG_CQE_INLINE);
4795		hr_reg_clear(qpc_mask, QPC_CQEIE);
4796
4797		hr_reg_write(context, QPC_CQEIS, 0);
4798		hr_reg_clear(qpc_mask, QPC_CQEIS);
4799	}
4800
4801	return 0;
4802}
4803
4804static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, int attr_mask,
4805				struct hns_roce_v2_qp_context *context,
4806				struct hns_roce_v2_qp_context *qpc_mask)
4807{
4808	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4809	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4810	struct ib_device *ibdev = &hr_dev->ib_dev;
4811	int ret;
4812
4813	/* Not support alternate path and path migration */
4814	if (attr_mask & (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) {
4815		ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
4816		return -EINVAL;
4817	}
4818
4819	ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask);
4820	if (ret) {
4821		ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret);
4822		return ret;
4823	}
4824
4825	/*
4826	 * Set some fields in context to zero, Because the default values
4827	 * of all fields in context are zero, we need not set them to 0 again.
4828	 * but we should set the relevant fields of context mask to 0.
4829	 */
4830	hr_reg_clear(qpc_mask, QPC_IRRL_SGE_IDX);
4831
4832	hr_reg_clear(qpc_mask, QPC_RX_ACK_MSN);
4833
4834	hr_reg_clear(qpc_mask, QPC_ACK_LAST_OPTYPE);
4835	hr_reg_clear(qpc_mask, QPC_IRRL_PSN_VLD);
4836	hr_reg_clear(qpc_mask, QPC_IRRL_PSN);
4837
4838	hr_reg_clear(qpc_mask, QPC_IRRL_TAIL_REAL);
4839
4840	hr_reg_clear(qpc_mask, QPC_RETRY_MSG_MSN);
4841
4842	hr_reg_clear(qpc_mask, QPC_RNR_RETRY_FLAG);
4843
4844	hr_reg_clear(qpc_mask, QPC_CHECK_FLG);
4845
4846	hr_reg_clear(qpc_mask, QPC_V2_IRRL_HEAD);
4847
4848	return 0;
4849}
4850
4851static int alloc_dip_entry(struct xarray *dip_xa, u32 qpn)
4852{
4853	struct hns_roce_dip *hr_dip;
4854	int ret;
4855
4856	hr_dip = xa_load(dip_xa, qpn);
4857	if (hr_dip)
4858		return 0;
4859
4860	hr_dip = kzalloc(sizeof(*hr_dip), GFP_KERNEL);
4861	if (!hr_dip)
4862		return -ENOMEM;
4863
4864	ret = xa_err(xa_store(dip_xa, qpn, hr_dip, GFP_KERNEL));
4865	if (ret)
4866		kfree(hr_dip);
4867
4868	return ret;
4869}
4870
4871static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
4872			   u32 *dip_idx)
4873{
4874	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4875	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4876	struct xarray *dip_xa = &hr_dev->qp_table.dip_xa;
4877	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4878	struct hns_roce_dip *hr_dip;
4879	unsigned long idx;
4880	int ret = 0;
4881
4882	ret = alloc_dip_entry(dip_xa, ibqp->qp_num);
4883	if (ret)
4884		return ret;
4885
4886	xa_lock(dip_xa);
4887
4888	xa_for_each(dip_xa, idx, hr_dip) {
4889		if (hr_dip->qp_cnt &&
4890		    !memcmp(grh->dgid.raw, hr_dip->dgid, GID_LEN_V2)) {
4891			*dip_idx = hr_dip->dip_idx;
4892			hr_dip->qp_cnt++;
4893			hr_qp->dip = hr_dip;
4894			goto out;
4895		}
4896	}
4897
4898	/* If no dgid is found, a new dip and a mapping between dgid and
4899	 * dip_idx will be created.
4900	 */
4901	xa_for_each(dip_xa, idx, hr_dip) {
4902		if (hr_dip->qp_cnt)
4903			continue;
4904
4905		*dip_idx = idx;
4906		memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
4907		hr_dip->dip_idx = idx;
4908		hr_dip->qp_cnt++;
4909		hr_qp->dip = hr_dip;
4910		break;
4911	}
4912
4913	/* This should never happen. */
4914	if (WARN_ON_ONCE(!hr_qp->dip))
4915		ret = -ENOSPC;
4916
4917out:
4918	xa_unlock(dip_xa);
4919	return ret;
4920}
4921
4922enum {
4923	CONG_DCQCN,
4924	CONG_WINDOW,
4925};
4926
4927enum {
4928	UNSUPPORT_CONG_LEVEL,
4929	SUPPORT_CONG_LEVEL,
4930};
4931
4932enum {
4933	CONG_LDCP,
4934	CONG_HC3,
4935};
4936
4937enum {
4938	DIP_INVALID,
4939	DIP_VALID,
4940};
4941
4942enum {
4943	WND_LIMIT,
4944	WND_UNLIMIT,
4945};
4946
4947static int check_cong_type(struct ib_qp *ibqp,
4948			   struct hns_roce_congestion_algorithm *cong_alg)
4949{
4950	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4951
4952	/* different congestion types match different configurations */
4953	switch (hr_qp->cong_type) {
4954	case CONG_TYPE_DCQCN:
4955		cong_alg->alg_sel = CONG_DCQCN;
4956		cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;
4957		cong_alg->dip_vld = DIP_INVALID;
4958		cong_alg->wnd_mode_sel = WND_LIMIT;
4959		break;
4960	case CONG_TYPE_LDCP:
4961		cong_alg->alg_sel = CONG_WINDOW;
4962		cong_alg->alg_sub_sel = CONG_LDCP;
4963		cong_alg->dip_vld = DIP_INVALID;
4964		cong_alg->wnd_mode_sel = WND_UNLIMIT;
4965		break;
4966	case CONG_TYPE_HC3:
4967		cong_alg->alg_sel = CONG_WINDOW;
4968		cong_alg->alg_sub_sel = CONG_HC3;
4969		cong_alg->dip_vld = DIP_INVALID;
4970		cong_alg->wnd_mode_sel = WND_LIMIT;
4971		break;
4972	case CONG_TYPE_DIP:
4973		cong_alg->alg_sel = CONG_DCQCN;
4974		cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;
4975		cong_alg->dip_vld = DIP_VALID;
4976		cong_alg->wnd_mode_sel = WND_LIMIT;
4977		break;
4978	default:
4979		hr_qp->cong_type = CONG_TYPE_DCQCN;
4980		cong_alg->alg_sel = CONG_DCQCN;
4981		cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;
4982		cong_alg->dip_vld = DIP_INVALID;
4983		cong_alg->wnd_mode_sel = WND_LIMIT;
4984		break;
4985	}
4986
4987	return 0;
4988}
4989
4990static int fill_cong_field(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
4991			   struct hns_roce_v2_qp_context *context,
4992			   struct hns_roce_v2_qp_context *qpc_mask)
4993{
4994	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4995	struct hns_roce_congestion_algorithm cong_field;
4996	struct ib_device *ibdev = ibqp->device;
4997	struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
4998	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4999	u32 dip_idx = 0;
5000	int ret;
5001
5002	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ||
5003	    grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE)
5004		return 0;
5005
5006	ret = check_cong_type(ibqp, &cong_field);
5007	if (ret)
5008		return ret;
5009
5010	hr_reg_write(context, QPC_CONG_ALGO_TMPL_ID, hr_dev->cong_algo_tmpl_id +
5011		     hr_qp->cong_type * HNS_ROCE_CONG_SIZE);
5012	hr_reg_clear(qpc_mask, QPC_CONG_ALGO_TMPL_ID);
5013	hr_reg_write(&context->ext, QPCEX_CONG_ALG_SEL, cong_field.alg_sel);
5014	hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SEL);
5015	hr_reg_write(&context->ext, QPCEX_CONG_ALG_SUB_SEL,
5016		     cong_field.alg_sub_sel);
5017	hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SUB_SEL);
5018	hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX_VLD, cong_field.dip_vld);
5019	hr_reg_clear(&qpc_mask->ext, QPCEX_DIP_CTX_IDX_VLD);
5020	hr_reg_write(&context->ext, QPCEX_SQ_RQ_NOT_FORBID_EN,
5021		     cong_field.wnd_mode_sel);
5022	hr_reg_clear(&qpc_mask->ext, QPCEX_SQ_RQ_NOT_FORBID_EN);
5023
5024	/* if dip is disabled, there is no need to set dip idx */
5025	if (cong_field.dip_vld == 0)
5026		return 0;
5027
5028	ret = get_dip_ctx_idx(ibqp, attr, &dip_idx);
5029	if (ret) {
5030		ibdev_err(ibdev, "failed to fill cong field, ret = %d.\n", ret);
5031		return ret;
5032	}
5033
5034	hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX, dip_idx);
5035	hr_reg_write(&qpc_mask->ext, QPCEX_DIP_CTX_IDX, 0);
5036
5037	return 0;
5038}
5039
5040static int hns_roce_hw_v2_get_dscp(struct hns_roce_dev *hr_dev, u8 dscp,
5041				   u8 *tc_mode, u8 *priority)
5042{
5043	struct hns_roce_v2_priv *priv = hr_dev->priv;
5044	struct hnae3_handle *handle = priv->handle;
5045	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
5046
5047	if (!ops->get_dscp_prio)
5048		return -EOPNOTSUPP;
5049
5050	return ops->get_dscp_prio(handle, dscp, tc_mode, priority);
5051}
5052
5053bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl)
5054{
5055	u32 max_sl;
5056
5057	max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev->caps.sl_num - 1);
5058	if (unlikely(sl > max_sl)) {
5059		ibdev_err_ratelimited(&hr_dev->ib_dev,
5060				      "failed to set SL(%u). Shouldn't be larger than %u.\n",
5061				      sl, max_sl);
5062		return false;
5063	}
5064
5065	return true;
5066}
5067
5068static int hns_roce_set_sl(struct ib_qp *ibqp,
5069			   const struct ib_qp_attr *attr,
5070			   struct hns_roce_v2_qp_context *context,
5071			   struct hns_roce_v2_qp_context *qpc_mask)
5072{
5073	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
5074	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5075	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5076	struct ib_device *ibdev = &hr_dev->ib_dev;
5077	int ret;
5078
5079	hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
5080
5081	if (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
5082		ret = hns_roce_hw_v2_get_dscp(hr_dev,
5083					      get_tclass(&attr->ah_attr.grh),
5084					      &hr_qp->tc_mode, &hr_qp->priority);
5085		if (ret && ret != -EOPNOTSUPP) {
5086			ibdev_err_ratelimited(ibdev,
5087					      "failed to get dscp, ret = %d.\n",
5088					      ret);
5089			return ret;
5090		}
5091
5092		if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP)
5093			hr_qp->sl = hr_qp->priority;
5094	}
5095
5096	if (!check_sl_valid(hr_dev, hr_qp->sl))
5097		return -EINVAL;
5098
5099	hr_reg_write(context, QPC_SL, hr_qp->sl);
5100	hr_reg_clear(qpc_mask, QPC_SL);
5101
5102	return 0;
5103}
5104
5105static int hns_roce_v2_set_path(struct ib_qp *ibqp,
5106				const struct ib_qp_attr *attr,
5107				int attr_mask,
5108				struct hns_roce_v2_qp_context *context,
5109				struct hns_roce_v2_qp_context *qpc_mask)
5110{
5111	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
5112	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5113	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5114	struct ib_device *ibdev = &hr_dev->ib_dev;
5115	const struct ib_gid_attr *gid_attr = NULL;
5116	u8 sl = rdma_ah_get_sl(&attr->ah_attr);
5117	int is_roce_protocol;
5118	u16 vlan_id = 0xffff;
5119	bool is_udp = false;
5120	u8 ib_port;
5121	u8 hr_port;
5122	int ret;
5123
5124	/*
5125	 * If free_mr_en of qp is set, it means that this qp comes from
5126	 * free mr. This qp will perform the loopback operation.
5127	 * In the loopback scenario, only sl needs to be set.
5128	 */
5129	if (hr_qp->free_mr_en) {
5130		if (!check_sl_valid(hr_dev, sl))
5131			return -EINVAL;
5132		hr_reg_write(context, QPC_SL, sl);
5133		hr_reg_clear(qpc_mask, QPC_SL);
5134		hr_qp->sl = sl;
5135		return 0;
5136	}
5137
5138	ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;
5139	hr_port = ib_port - 1;
5140	is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
5141			   rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
5142
5143	if (is_roce_protocol) {
5144		gid_attr = attr->ah_attr.grh.sgid_attr;
5145		ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
5146		if (ret)
5147			return ret;
5148
5149		is_udp = (gid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP);
5150	}
5151
5152	/* Only HIP08 needs to set the vlan_en bits in QPC */
5153	if (vlan_id < VLAN_N_VID &&
5154	    hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
5155		hr_reg_enable(context, QPC_RQ_VLAN_EN);
5156		hr_reg_clear(qpc_mask, QPC_RQ_VLAN_EN);
5157		hr_reg_enable(context, QPC_SQ_VLAN_EN);
5158		hr_reg_clear(qpc_mask, QPC_SQ_VLAN_EN);
5159	}
5160
5161	hr_reg_write(context, QPC_VLAN_ID, vlan_id);
5162	hr_reg_clear(qpc_mask, QPC_VLAN_ID);
5163
5164	if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
5165		ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",
5166			  grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
5167		return -EINVAL;
5168	}
5169
5170	if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
5171		ibdev_err(ibdev, "ah attr is not RDMA roce type\n");
5172		return -EINVAL;
5173	}
5174
5175	hr_reg_write(context, QPC_UDPSPN,
5176		     is_udp ? rdma_get_udp_sport(grh->flow_label, ibqp->qp_num,
5177						 attr->dest_qp_num) :
5178				    0);
5179
5180	hr_reg_clear(qpc_mask, QPC_UDPSPN);
5181
5182	hr_reg_write(context, QPC_GMV_IDX, grh->sgid_index);
5183
5184	hr_reg_clear(qpc_mask, QPC_GMV_IDX);
5185
5186	hr_reg_write(context, QPC_HOPLIMIT, grh->hop_limit);
5187	hr_reg_clear(qpc_mask, QPC_HOPLIMIT);
5188
5189	ret = fill_cong_field(ibqp, attr, context, qpc_mask);
5190	if (ret)
5191		return ret;
5192
5193	hr_reg_write(context, QPC_TC, get_tclass(&attr->ah_attr.grh));
5194	hr_reg_clear(qpc_mask, QPC_TC);
5195
5196	hr_reg_write(context, QPC_FL, grh->flow_label);
5197	hr_reg_clear(qpc_mask, QPC_FL);
5198	memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
5199	memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
5200
5201	return  hns_roce_set_sl(ibqp, attr, context, qpc_mask);
5202}
5203
5204static bool check_qp_state(enum ib_qp_state cur_state,
5205			   enum ib_qp_state new_state)
5206{
5207	static const bool sm[][IB_QPS_ERR + 1] = {
5208		[IB_QPS_RESET] = { [IB_QPS_RESET] = true,
5209				   [IB_QPS_INIT] = true },
5210		[IB_QPS_INIT] = { [IB_QPS_RESET] = true,
5211				  [IB_QPS_INIT] = true,
5212				  [IB_QPS_RTR] = true,
5213				  [IB_QPS_ERR] = true },
5214		[IB_QPS_RTR] = { [IB_QPS_RESET] = true,
5215				 [IB_QPS_RTS] = true,
5216				 [IB_QPS_ERR] = true },
5217		[IB_QPS_RTS] = { [IB_QPS_RESET] = true,
5218				 [IB_QPS_RTS] = true,
5219				 [IB_QPS_ERR] = true },
5220		[IB_QPS_SQD] = {},
5221		[IB_QPS_SQE] = {},
5222		[IB_QPS_ERR] = { [IB_QPS_RESET] = true,
5223				 [IB_QPS_ERR] = true }
5224	};
5225
5226	return sm[cur_state][new_state];
5227}
5228
5229static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
5230				      const struct ib_qp_attr *attr,
5231				      int attr_mask,
5232				      enum ib_qp_state cur_state,
5233				      enum ib_qp_state new_state,
5234				      struct hns_roce_v2_qp_context *context,
5235				      struct hns_roce_v2_qp_context *qpc_mask,
5236				      struct ib_udata *udata)
5237{
5238	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5239	int ret = 0;
5240
5241	if (!check_qp_state(cur_state, new_state))
5242		return -EINVAL;
5243
5244	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
5245		memset(qpc_mask, 0, hr_dev->caps.qpc_sz);
5246		modify_qp_reset_to_init(ibqp, context);
5247	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
5248		modify_qp_init_to_init(ibqp, context, qpc_mask);
5249	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
5250		ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
5251					    qpc_mask, udata);
5252	} else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
5253		ret = modify_qp_rtr_to_rts(ibqp, attr_mask, context, qpc_mask);
5254	}
5255
5256	return ret;
5257}
5258
5259static bool check_qp_timeout_cfg_range(struct hns_roce_dev *hr_dev, u8 *timeout)
5260{
5261#define QP_ACK_TIMEOUT_MAX_HIP08 20
5262#define QP_ACK_TIMEOUT_MAX 31
5263
5264	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
5265		if (*timeout > QP_ACK_TIMEOUT_MAX_HIP08) {
5266			ibdev_warn(&hr_dev->ib_dev,
5267				   "local ACK timeout shall be 0 to 20.\n");
5268			return false;
5269		}
5270		*timeout += HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08;
5271	} else if (hr_dev->pci_dev->revision > PCI_REVISION_ID_HIP08) {
5272		if (*timeout > QP_ACK_TIMEOUT_MAX) {
5273			ibdev_warn(&hr_dev->ib_dev,
5274				   "local ACK timeout shall be 0 to 31.\n");
5275			return false;
5276		}
5277	}
5278
5279	return true;
5280}
5281
5282static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
5283				      const struct ib_qp_attr *attr,
5284				      int attr_mask,
5285				      struct hns_roce_v2_qp_context *context,
5286				      struct hns_roce_v2_qp_context *qpc_mask)
5287{
5288	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5289	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5290	int ret = 0;
5291	u8 timeout;
5292
5293	if (attr_mask & IB_QP_AV) {
5294		ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,
5295					   qpc_mask);
5296		if (ret)
5297			return ret;
5298	}
5299
5300	if (attr_mask & IB_QP_TIMEOUT) {
5301		timeout = attr->timeout;
5302		if (check_qp_timeout_cfg_range(hr_dev, &timeout)) {
5303			hr_reg_write(context, QPC_AT, timeout);
5304			hr_reg_clear(qpc_mask, QPC_AT);
5305		}
5306	}
5307
5308	if (attr_mask & IB_QP_RETRY_CNT) {
5309		hr_reg_write(context, QPC_RETRY_NUM_INIT, attr->retry_cnt);
5310		hr_reg_clear(qpc_mask, QPC_RETRY_NUM_INIT);
5311
5312		hr_reg_write(context, QPC_RETRY_CNT, attr->retry_cnt);
5313		hr_reg_clear(qpc_mask, QPC_RETRY_CNT);
5314	}
5315
5316	if (attr_mask & IB_QP_RNR_RETRY) {
5317		hr_reg_write(context, QPC_RNR_NUM_INIT, attr->rnr_retry);
5318		hr_reg_clear(qpc_mask, QPC_RNR_NUM_INIT);
5319
5320		hr_reg_write(context, QPC_RNR_CNT, attr->rnr_retry);
5321		hr_reg_clear(qpc_mask, QPC_RNR_CNT);
5322	}
5323
5324	if (attr_mask & IB_QP_SQ_PSN) {
5325		hr_reg_write(context, QPC_SQ_CUR_PSN, attr->sq_psn);
5326		hr_reg_clear(qpc_mask, QPC_SQ_CUR_PSN);
5327
5328		hr_reg_write(context, QPC_SQ_MAX_PSN, attr->sq_psn);
5329		hr_reg_clear(qpc_mask, QPC_SQ_MAX_PSN);
5330
5331		hr_reg_write(context, QPC_RETRY_MSG_PSN_L, attr->sq_psn);
5332		hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_L);
5333
5334		hr_reg_write(context, QPC_RETRY_MSG_PSN_H,
5335			     attr->sq_psn >> RETRY_MSG_PSN_SHIFT);
5336		hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_H);
5337
5338		hr_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr->sq_psn);
5339		hr_reg_clear(qpc_mask, QPC_RETRY_MSG_FPKT_PSN);
5340
5341		hr_reg_write(context, QPC_RX_ACK_EPSN, attr->sq_psn);
5342		hr_reg_clear(qpc_mask, QPC_RX_ACK_EPSN);
5343	}
5344
5345	if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
5346	     attr->max_dest_rd_atomic) {
5347		hr_reg_write(context, QPC_RR_MAX,
5348			     fls(attr->max_dest_rd_atomic - 1));
5349		hr_reg_clear(qpc_mask, QPC_RR_MAX);
5350	}
5351
5352	if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
5353		hr_reg_write(context, QPC_SR_MAX, fls(attr->max_rd_atomic - 1));
5354		hr_reg_clear(qpc_mask, QPC_SR_MAX);
5355	}
5356
5357	if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
5358		set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
5359
5360	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
5361		hr_reg_write(context, QPC_MIN_RNR_TIME,
5362			    hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ?
5363			    HNS_ROCE_RNR_TIMER_10NS : attr->min_rnr_timer);
5364		hr_reg_clear(qpc_mask, QPC_MIN_RNR_TIME);
5365	}
5366
5367	if (attr_mask & IB_QP_RQ_PSN) {
5368		hr_reg_write(context, QPC_RX_REQ_EPSN, attr->rq_psn);
5369		hr_reg_clear(qpc_mask, QPC_RX_REQ_EPSN);
5370
5371		hr_reg_write(context, QPC_RAQ_PSN, attr->rq_psn - 1);
5372		hr_reg_clear(qpc_mask, QPC_RAQ_PSN);
5373	}
5374
5375	if (attr_mask & IB_QP_QKEY) {
5376		context->qkey_xrcd = cpu_to_le32(attr->qkey);
5377		qpc_mask->qkey_xrcd = 0;
5378		hr_qp->qkey = attr->qkey;
5379	}
5380
5381	return ret;
5382}
5383
5384static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,
5385					  const struct ib_qp_attr *attr,
5386					  int attr_mask)
5387{
5388	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5389	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5390
5391	if (attr_mask & IB_QP_ACCESS_FLAGS)
5392		hr_qp->atomic_rd_en = attr->qp_access_flags;
5393
5394	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
5395		hr_qp->resp_depth = attr->max_dest_rd_atomic;
5396	if (attr_mask & IB_QP_PORT) {
5397		hr_qp->port = attr->port_num - 1;
5398		hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
5399	}
5400}
5401
5402static void clear_qp(struct hns_roce_qp *hr_qp)
5403{
5404	struct ib_qp *ibqp = &hr_qp->ibqp;
5405
5406	if (ibqp->send_cq)
5407		hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),
5408				     hr_qp->qpn, NULL);
5409
5410	if (ibqp->recv_cq  && ibqp->recv_cq != ibqp->send_cq)
5411		hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq),
5412				     hr_qp->qpn, ibqp->srq ?
5413				     to_hr_srq(ibqp->srq) : NULL);
5414
5415	if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
5416		*hr_qp->rdb.db_record = 0;
5417
5418	hr_qp->rq.head = 0;
5419	hr_qp->rq.tail = 0;
5420	hr_qp->sq.head = 0;
5421	hr_qp->sq.tail = 0;
5422	hr_qp->next_sge = 0;
5423}
5424
5425static void v2_set_flushed_fields(struct ib_qp *ibqp,
5426				  struct hns_roce_v2_qp_context *context,
5427				  struct hns_roce_v2_qp_context *qpc_mask)
5428{
5429	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5430	unsigned long sq_flag = 0;
5431	unsigned long rq_flag = 0;
5432
5433	if (ibqp->qp_type == IB_QPT_XRC_TGT)
5434		return;
5435
5436	spin_lock_irqsave(&hr_qp->sq.lock, sq_flag);
5437	trace_hns_sq_flush_cqe(hr_qp->qpn, hr_qp->sq.head, TRACE_SQ);
5438	hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head);
5439	hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX);
5440	hr_qp->state = IB_QPS_ERR;
5441	spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag);
5442
5443	if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */
5444		return;
5445
5446	spin_lock_irqsave(&hr_qp->rq.lock, rq_flag);
5447	trace_hns_rq_flush_cqe(hr_qp->qpn, hr_qp->rq.head, TRACE_RQ);
5448	hr_reg_write(context, QPC_RQ_PRODUCER_IDX, hr_qp->rq.head);
5449	hr_reg_clear(qpc_mask, QPC_RQ_PRODUCER_IDX);
5450	spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag);
5451}
5452
5453static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
5454				 const struct ib_qp_attr *attr,
5455				 int attr_mask, enum ib_qp_state cur_state,
5456				 enum ib_qp_state new_state, struct ib_udata *udata)
5457{
5458	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5459	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5460	struct hns_roce_v2_qp_context *context;
5461	struct hns_roce_v2_qp_context *qpc_mask;
5462	struct ib_device *ibdev = &hr_dev->ib_dev;
5463	int ret = -ENOMEM;
5464
5465	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
5466		return -EOPNOTSUPP;
5467
5468	/*
5469	 * In v2 engine, software pass context and context mask to hardware
5470	 * when modifying qp. If software need modify some fields in context,
5471	 * we should set all bits of the relevant fields in context mask to
5472	 * 0 at the same time, else set them to 0x1.
5473	 */
5474	context = kvzalloc(sizeof(*context), GFP_KERNEL);
5475	qpc_mask = kvzalloc(sizeof(*qpc_mask), GFP_KERNEL);
5476	if (!context || !qpc_mask)
5477		goto out;
5478
5479	memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz);
5480
5481	ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
5482					 new_state, context, qpc_mask, udata);
5483	if (ret)
5484		goto out;
5485
5486	/* When QP state is err, SQ and RQ WQE should be flushed */
5487	if (new_state == IB_QPS_ERR)
5488		v2_set_flushed_fields(ibqp, context, qpc_mask);
5489
5490	/* Configure the optional fields */
5491	ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,
5492					 qpc_mask);
5493	if (ret)
5494		goto out;
5495
5496	hr_reg_write_bool(context, QPC_INV_CREDIT,
5497			  to_hr_qp_type(hr_qp->ibqp.qp_type) == SERV_TYPE_XRC ||
5498			  ibqp->srq);
5499	hr_reg_clear(qpc_mask, QPC_INV_CREDIT);
5500
5501	/* Every status migrate must change state */
5502	hr_reg_write(context, QPC_QP_ST, new_state);
5503	hr_reg_clear(qpc_mask, QPC_QP_ST);
5504
5505	/* SW pass context to HW */
5506	ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp);
5507	if (ret) {
5508		ibdev_err_ratelimited(ibdev, "failed to modify QP, ret = %d.\n", ret);
5509		goto out;
5510	}
5511
5512	hr_qp->state = new_state;
5513
5514	hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);
5515
5516	if (new_state == IB_QPS_RESET && !ibqp->uobject)
5517		clear_qp(hr_qp);
5518
5519out:
5520	kvfree(qpc_mask);
5521	kvfree(context);
5522	return ret;
5523}
5524
5525static int to_ib_qp_st(enum hns_roce_v2_qp_state state)
5526{
5527	static const enum ib_qp_state map[] = {
5528		[HNS_ROCE_QP_ST_RST] = IB_QPS_RESET,
5529		[HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT,
5530		[HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR,
5531		[HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS,
5532		[HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD,
5533		[HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE,
5534		[HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR,
5535		[HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD
5536	};
5537
5538	return (state < ARRAY_SIZE(map)) ? map[state] : -1;
5539}
5540
5541static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, u32 qpn,
5542				 void *buffer)
5543{
5544	struct hns_roce_cmd_mailbox *mailbox;
5545	int ret;
5546
5547	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5548	if (IS_ERR(mailbox))
5549		return PTR_ERR(mailbox);
5550
5551	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_QPC,
5552				qpn);
5553	if (ret)
5554		goto out;
5555
5556	memcpy(buffer, mailbox->buf, hr_dev->caps.qpc_sz);
5557
5558out:
5559	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5560	return ret;
5561}
5562
5563static int hns_roce_v2_query_srqc(struct hns_roce_dev *hr_dev, u32 srqn,
5564				 void *buffer)
5565{
5566	struct hns_roce_srq_context *context;
5567	struct hns_roce_cmd_mailbox *mailbox;
5568	int ret;
5569
5570	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5571	if (IS_ERR(mailbox))
5572		return PTR_ERR(mailbox);
5573
5574	context = mailbox->buf;
5575	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SRQC,
5576				srqn);
5577	if (ret)
5578		goto out;
5579
5580	memcpy(buffer, context, sizeof(*context));
5581
5582out:
5583	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5584	return ret;
5585}
5586
5587static int hns_roce_v2_query_sccc(struct hns_roce_dev *hr_dev, u32 sccn,
5588				  void *buffer)
5589{
5590	struct hns_roce_v2_scc_context *context;
5591	struct hns_roce_cmd_mailbox *mailbox;
5592	int ret;
5593
5594	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5595	if (IS_ERR(mailbox))
5596		return PTR_ERR(mailbox);
5597
5598	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SCCC,
5599				sccn);
5600	if (ret)
5601		goto out;
5602
5603	context = mailbox->buf;
5604	memcpy(buffer, context, sizeof(*context));
5605
5606out:
5607	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5608	return ret;
5609}
5610
5611static u8 get_qp_timeout_attr(struct hns_roce_dev *hr_dev,
5612			      struct hns_roce_v2_qp_context *context)
5613{
5614	u8 timeout;
5615
5616	timeout = (u8)hr_reg_read(context, QPC_AT);
5617	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
5618		timeout -= HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08;
5619
5620	return timeout;
5621}
5622
5623static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
5624				int qp_attr_mask,
5625				struct ib_qp_init_attr *qp_init_attr)
5626{
5627	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5628	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5629	struct hns_roce_v2_qp_context context = {};
5630	struct ib_device *ibdev = &hr_dev->ib_dev;
5631	int tmp_qp_state;
5632	int state;
5633	int ret;
5634
5635	memset(qp_attr, 0, sizeof(*qp_attr));
5636	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
5637
5638	mutex_lock(&hr_qp->mutex);
5639
5640	if (hr_qp->state == IB_QPS_RESET) {
5641		qp_attr->qp_state = IB_QPS_RESET;
5642		ret = 0;
5643		goto done;
5644	}
5645
5646	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context);
5647	if (ret) {
5648		ibdev_err_ratelimited(ibdev,
5649				      "failed to query QPC, ret = %d.\n",
5650				      ret);
5651		ret = -EINVAL;
5652		goto out;
5653	}
5654
5655	state = hr_reg_read(&context, QPC_QP_ST);
5656	tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
5657	if (tmp_qp_state == -1) {
5658		ibdev_err_ratelimited(ibdev, "Illegal ib_qp_state\n");
5659		ret = -EINVAL;
5660		goto out;
5661	}
5662	hr_qp->state = (u8)tmp_qp_state;
5663	qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
5664	qp_attr->path_mtu = (enum ib_mtu)hr_reg_read(&context, QPC_MTU);
5665	qp_attr->path_mig_state = IB_MIG_ARMED;
5666	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
5667	if (hr_qp->ibqp.qp_type == IB_QPT_UD)
5668		qp_attr->qkey = le32_to_cpu(context.qkey_xrcd);
5669
5670	qp_attr->rq_psn = hr_reg_read(&context, QPC_RX_REQ_EPSN);
5671	qp_attr->sq_psn = (u32)hr_reg_read(&context, QPC_SQ_CUR_PSN);
5672	qp_attr->dest_qp_num = hr_reg_read(&context, QPC_DQPN);
5673	qp_attr->qp_access_flags =
5674		((hr_reg_read(&context, QPC_RRE)) << V2_QP_RRE_S) |
5675		((hr_reg_read(&context, QPC_RWE)) << V2_QP_RWE_S) |
5676		((hr_reg_read(&context, QPC_ATE)) << V2_QP_ATE_S);
5677
5678	if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
5679	    hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
5680	    hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) {
5681		struct ib_global_route *grh =
5682			rdma_ah_retrieve_grh(&qp_attr->ah_attr);
5683
5684		rdma_ah_set_sl(&qp_attr->ah_attr,
5685			       hr_reg_read(&context, QPC_SL));
5686		rdma_ah_set_port_num(&qp_attr->ah_attr, hr_qp->port + 1);
5687		rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH);
5688		grh->flow_label = hr_reg_read(&context, QPC_FL);
5689		grh->sgid_index = hr_reg_read(&context, QPC_GMV_IDX);
5690		grh->hop_limit = hr_reg_read(&context, QPC_HOPLIMIT);
5691		grh->traffic_class = hr_reg_read(&context, QPC_TC);
5692
5693		memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw));
5694	}
5695
5696	qp_attr->port_num = hr_qp->port + 1;
5697	qp_attr->sq_draining = 0;
5698	qp_attr->max_rd_atomic = 1 << hr_reg_read(&context, QPC_SR_MAX);
5699	qp_attr->max_dest_rd_atomic = 1 << hr_reg_read(&context, QPC_RR_MAX);
5700
5701	qp_attr->min_rnr_timer = (u8)hr_reg_read(&context, QPC_MIN_RNR_TIME);
5702	qp_attr->timeout = get_qp_timeout_attr(hr_dev, &context);
5703	qp_attr->retry_cnt = hr_reg_read(&context, QPC_RETRY_NUM_INIT);
5704	qp_attr->rnr_retry = hr_reg_read(&context, QPC_RNR_NUM_INIT);
5705
5706done:
5707	qp_attr->cur_qp_state = qp_attr->qp_state;
5708	qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
5709	qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
5710	qp_attr->cap.max_inline_data = hr_qp->max_inline_data;
5711
5712	qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
5713	qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
5714
5715	qp_init_attr->qp_context = ibqp->qp_context;
5716	qp_init_attr->qp_type = ibqp->qp_type;
5717	qp_init_attr->recv_cq = ibqp->recv_cq;
5718	qp_init_attr->send_cq = ibqp->send_cq;
5719	qp_init_attr->srq = ibqp->srq;
5720	qp_init_attr->cap = qp_attr->cap;
5721	qp_init_attr->sq_sig_type = hr_qp->sq_signal_bits;
5722
5723out:
5724	mutex_unlock(&hr_qp->mutex);
5725	return ret;
5726}
5727
5728static inline int modify_qp_is_ok(struct hns_roce_qp *hr_qp)
5729{
5730	return ((hr_qp->ibqp.qp_type == IB_QPT_RC ||
5731		 hr_qp->ibqp.qp_type == IB_QPT_UD ||
5732		 hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
5733		 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) &&
5734		hr_qp->state != IB_QPS_RESET);
5735}
5736
5737static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
5738					 struct hns_roce_qp *hr_qp,
5739					 struct ib_udata *udata)
5740{
5741	struct ib_device *ibdev = &hr_dev->ib_dev;
5742	struct hns_roce_cq *send_cq, *recv_cq;
5743	unsigned long flags;
5744	int ret = 0;
5745
5746	if (modify_qp_is_ok(hr_qp)) {
5747		/* Modify qp to reset before destroying qp */
5748		ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
5749					    hr_qp->state, IB_QPS_RESET, udata);
5750		if (ret)
5751			ibdev_err_ratelimited(ibdev,
5752					      "failed to modify QP to RST, ret = %d.\n",
5753					      ret);
5754	}
5755
5756	send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL;
5757	recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL;
5758
5759	spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
5760	hns_roce_lock_cqs(send_cq, recv_cq);
5761
5762	if (!udata) {
5763		if (recv_cq)
5764			__hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn,
5765					       (hr_qp->ibqp.srq ?
5766						to_hr_srq(hr_qp->ibqp.srq) :
5767						NULL));
5768
5769		if (send_cq && send_cq != recv_cq)
5770			__hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
5771	}
5772
5773	hns_roce_qp_remove(hr_dev, hr_qp);
5774
5775	hns_roce_unlock_cqs(send_cq, recv_cq);
5776	spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
5777
5778	return ret;
5779}
5780
5781static void put_dip_ctx_idx(struct hns_roce_dev *hr_dev,
5782			    struct hns_roce_qp *hr_qp)
5783{
5784	struct hns_roce_dip *hr_dip = hr_qp->dip;
5785
5786	if (!hr_dip)
5787		return;
5788
5789	xa_lock(&hr_dev->qp_table.dip_xa);
5790
5791	hr_dip->qp_cnt--;
5792	if (!hr_dip->qp_cnt)
5793		memset(hr_dip->dgid, 0, GID_LEN_V2);
5794
5795	xa_unlock(&hr_dev->qp_table.dip_xa);
5796}
5797
5798int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
5799{
5800	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5801	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5802	unsigned long flags;
5803	int ret;
5804
5805	/* Make sure flush_cqe() is completed */
5806	spin_lock_irqsave(&hr_qp->flush_lock, flags);
5807	set_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag);
5808	spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
5809	flush_work(&hr_qp->flush_work.work);
5810
5811	if (hr_qp->cong_type == CONG_TYPE_DIP)
5812		put_dip_ctx_idx(hr_dev, hr_qp);
5813
5814	ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
5815	if (ret)
5816		ibdev_err_ratelimited(&hr_dev->ib_dev,
5817				      "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n",
5818				      hr_qp->qpn, ret);
5819
5820	hns_roce_qp_destroy(hr_dev, hr_qp, udata);
5821
5822	return 0;
5823}
5824
5825static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
5826					    struct hns_roce_qp *hr_qp)
5827{
5828	struct ib_device *ibdev = &hr_dev->ib_dev;
5829	struct hns_roce_sccc_clr_done *resp;
5830	struct hns_roce_sccc_clr *clr;
5831	struct hns_roce_cmq_desc desc;
5832	int ret, i;
5833
5834	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
5835		return 0;
5836
5837	mutex_lock(&hr_dev->qp_table.scc_mutex);
5838
5839	/* set scc ctx clear done flag */
5840	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
5841	ret =  hns_roce_cmq_send(hr_dev, &desc, 1);
5842	if (ret) {
5843		ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret);
5844		goto out;
5845	}
5846
5847	/* clear scc context */
5848	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
5849	clr = (struct hns_roce_sccc_clr *)desc.data;
5850	clr->qpn = cpu_to_le32(hr_qp->qpn);
5851	ret =  hns_roce_cmq_send(hr_dev, &desc, 1);
5852	if (ret) {
5853		ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret);
5854		goto out;
5855	}
5856
5857	/* query scc context clear is done or not */
5858	resp = (struct hns_roce_sccc_clr_done *)desc.data;
5859	for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
5860		hns_roce_cmq_setup_basic_desc(&desc,
5861					      HNS_ROCE_OPC_QUERY_SCCC, true);
5862		ret = hns_roce_cmq_send(hr_dev, &desc, 1);
5863		if (ret) {
5864			ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n",
5865				  ret);
5866			goto out;
5867		}
5868
5869		if (resp->clr_done)
5870			goto out;
5871
5872		msleep(20);
5873	}
5874
5875	ibdev_err(ibdev, "query SCC clr done flag overtime.\n");
5876	ret = -ETIMEDOUT;
5877
5878out:
5879	mutex_unlock(&hr_dev->qp_table.scc_mutex);
5880	return ret;
5881}
5882
5883#define DMA_IDX_SHIFT 3
5884#define DMA_WQE_SHIFT 3
5885
5886static int hns_roce_v2_write_srqc_index_queue(struct hns_roce_srq *srq,
5887					      struct hns_roce_srq_context *ctx)
5888{
5889	struct hns_roce_idx_que *idx_que = &srq->idx_que;
5890	struct ib_device *ibdev = srq->ibsrq.device;
5891	struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
5892	u64 mtts_idx[MTT_MIN_COUNT] = {};
5893	dma_addr_t dma_handle_idx;
5894	int ret;
5895
5896	/* Get physical address of idx que buf */
5897	ret = hns_roce_mtr_find(hr_dev, &idx_que->mtr, 0, mtts_idx,
5898				ARRAY_SIZE(mtts_idx));
5899	if (ret) {
5900		ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n",
5901			  ret);
5902		return ret;
5903	}
5904
5905	dma_handle_idx = hns_roce_get_mtr_ba(&idx_que->mtr);
5906
5907	hr_reg_write(ctx, SRQC_IDX_HOP_NUM,
5908		     to_hr_hem_hopnum(hr_dev->caps.idx_hop_num, srq->wqe_cnt));
5909
5910	hr_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx >> DMA_IDX_SHIFT);
5911	hr_reg_write(ctx, SRQC_IDX_BT_BA_H,
5912		     upper_32_bits(dma_handle_idx >> DMA_IDX_SHIFT));
5913
5914	hr_reg_write(ctx, SRQC_IDX_BA_PG_SZ,
5915		     to_hr_hw_page_shift(idx_que->mtr.hem_cfg.ba_pg_shift));
5916	hr_reg_write(ctx, SRQC_IDX_BUF_PG_SZ,
5917		     to_hr_hw_page_shift(idx_que->mtr.hem_cfg.buf_pg_shift));
5918
5919	hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L,
5920		     to_hr_hw_page_addr(mtts_idx[0]));
5921	hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H,
5922		     upper_32_bits(to_hr_hw_page_addr(mtts_idx[0])));
5923
5924	hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L,
5925		     to_hr_hw_page_addr(mtts_idx[1]));
5926	hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H,
5927		     upper_32_bits(to_hr_hw_page_addr(mtts_idx[1])));
5928
5929	return 0;
5930}
5931
5932static int hns_roce_v2_write_srqc(struct hns_roce_srq *srq, void *mb_buf)
5933{
5934	struct ib_device *ibdev = srq->ibsrq.device;
5935	struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
5936	struct hns_roce_srq_context *ctx = mb_buf;
5937	u64 mtts_wqe[MTT_MIN_COUNT] = {};
5938	dma_addr_t dma_handle_wqe;
5939	int ret;
5940
5941	memset(ctx, 0, sizeof(*ctx));
5942
5943	/* Get the physical address of srq buf */
5944	ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe,
5945				ARRAY_SIZE(mtts_wqe));
5946	if (ret) {
5947		ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n",
5948			  ret);
5949		return ret;
5950	}
5951
5952	dma_handle_wqe = hns_roce_get_mtr_ba(&srq->buf_mtr);
5953
5954	hr_reg_write(ctx, SRQC_SRQ_ST, 1);
5955	hr_reg_write_bool(ctx, SRQC_SRQ_TYPE,
5956			  srq->ibsrq.srq_type == IB_SRQT_XRC);
5957	hr_reg_write(ctx, SRQC_PD, to_hr_pd(srq->ibsrq.pd)->pdn);
5958	hr_reg_write(ctx, SRQC_SRQN, srq->srqn);
5959	hr_reg_write(ctx, SRQC_XRCD, srq->xrcdn);
5960	hr_reg_write(ctx, SRQC_XRC_CQN, srq->cqn);
5961	hr_reg_write(ctx, SRQC_SHIFT, ilog2(srq->wqe_cnt));
5962	hr_reg_write(ctx, SRQC_RQWS,
5963		     srq->max_gs <= 0 ? 0 : fls(srq->max_gs - 1));
5964
5965	hr_reg_write(ctx, SRQC_WQE_HOP_NUM,
5966		     to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num,
5967				      srq->wqe_cnt));
5968
5969	hr_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe >> DMA_WQE_SHIFT);
5970	hr_reg_write(ctx, SRQC_WQE_BT_BA_H,
5971		     upper_32_bits(dma_handle_wqe >> DMA_WQE_SHIFT));
5972
5973	hr_reg_write(ctx, SRQC_WQE_BA_PG_SZ,
5974		     to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift));
5975	hr_reg_write(ctx, SRQC_WQE_BUF_PG_SZ,
5976		     to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift));
5977
5978	if (srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB) {
5979		hr_reg_enable(ctx, SRQC_DB_RECORD_EN);
5980		hr_reg_write(ctx, SRQC_DB_RECORD_ADDR_L,
5981			     lower_32_bits(srq->rdb.dma) >> 1);
5982		hr_reg_write(ctx, SRQC_DB_RECORD_ADDR_H,
5983			     upper_32_bits(srq->rdb.dma));
5984	}
5985
5986	return hns_roce_v2_write_srqc_index_queue(srq, ctx);
5987}
5988
5989static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
5990				  struct ib_srq_attr *srq_attr,
5991				  enum ib_srq_attr_mask srq_attr_mask,
5992				  struct ib_udata *udata)
5993{
5994	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5995	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5996	struct hns_roce_srq_context *srq_context;
5997	struct hns_roce_srq_context *srqc_mask;
5998	struct hns_roce_cmd_mailbox *mailbox;
5999	int ret = 0;
6000
6001	/* Resizing SRQs is not supported yet */
6002	if (srq_attr_mask & IB_SRQ_MAX_WR) {
6003		ret = -EOPNOTSUPP;
6004		goto out;
6005	}
6006
6007	if (srq_attr_mask & IB_SRQ_LIMIT) {
6008		if (srq_attr->srq_limit > srq->wqe_cnt) {
6009			ret = -EINVAL;
6010			goto out;
6011		}
6012
6013		mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6014		if (IS_ERR(mailbox)) {
6015			ret = PTR_ERR(mailbox);
6016			goto out;
6017		}
6018
6019		srq_context = mailbox->buf;
6020		srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
6021
6022		memset(srqc_mask, 0xff, sizeof(*srqc_mask));
6023
6024		hr_reg_write(srq_context, SRQC_LIMIT_WL, srq_attr->srq_limit);
6025		hr_reg_clear(srqc_mask, SRQC_LIMIT_WL);
6026
6027		ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,
6028					HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn);
6029		hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6030		if (ret)
6031			ibdev_err(&hr_dev->ib_dev,
6032				  "failed to handle cmd of modifying SRQ, ret = %d.\n",
6033				  ret);
6034	}
6035
6036out:
6037	if (ret)
6038		atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_SRQ_MODIFY_ERR_CNT]);
6039
6040	return ret;
6041}
6042
6043static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
6044{
6045	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6046	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6047	struct hns_roce_srq_context *srq_context;
6048	struct hns_roce_cmd_mailbox *mailbox;
6049	int ret;
6050
6051	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6052	if (IS_ERR(mailbox))
6053		return PTR_ERR(mailbox);
6054
6055	srq_context = mailbox->buf;
6056	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
6057				HNS_ROCE_CMD_QUERY_SRQC, srq->srqn);
6058	if (ret) {
6059		ibdev_err(&hr_dev->ib_dev,
6060			  "failed to process cmd of querying SRQ, ret = %d.\n",
6061			  ret);
6062		goto out;
6063	}
6064
6065	attr->srq_limit = hr_reg_read(srq_context, SRQC_LIMIT_WL);
6066	attr->max_wr = srq->wqe_cnt;
6067	attr->max_sge = srq->max_gs - srq->rsv_sge;
6068
6069out:
6070	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6071	return ret;
6072}
6073
6074static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
6075{
6076	struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
6077	struct hns_roce_v2_cq_context *cq_context;
6078	struct hns_roce_cq *hr_cq = to_hr_cq(cq);
6079	struct hns_roce_v2_cq_context *cqc_mask;
6080	struct hns_roce_cmd_mailbox *mailbox;
6081	int ret;
6082
6083	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6084	ret = PTR_ERR_OR_ZERO(mailbox);
6085	if (ret)
6086		goto err_out;
6087
6088	cq_context = mailbox->buf;
6089	cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;
6090
6091	memset(cqc_mask, 0xff, sizeof(*cqc_mask));
6092
6093	hr_reg_write(cq_context, CQC_CQ_MAX_CNT, cq_count);
6094	hr_reg_clear(cqc_mask, CQC_CQ_MAX_CNT);
6095
6096	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
6097		if (cq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) {
6098			dev_info(hr_dev->dev,
6099				 "cq_period(%u) reached the upper limit, adjusted to 65.\n",
6100				 cq_period);
6101			cq_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08;
6102		}
6103		cq_period *= HNS_ROCE_CLOCK_ADJUST;
6104	}
6105	hr_reg_write(cq_context, CQC_CQ_PERIOD, cq_period);
6106	hr_reg_clear(cqc_mask, CQC_CQ_PERIOD);
6107
6108	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,
6109				HNS_ROCE_CMD_MODIFY_CQC, hr_cq->cqn);
6110	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6111	if (ret)
6112		ibdev_err_ratelimited(&hr_dev->ib_dev,
6113				      "failed to process cmd when modifying CQ, ret = %d.\n",
6114				      ret);
6115
6116err_out:
6117	if (ret)
6118		atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CQ_MODIFY_ERR_CNT]);
6119
6120	return ret;
6121}
6122
6123static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,
6124				 void *buffer)
6125{
6126	struct hns_roce_v2_cq_context *context;
6127	struct hns_roce_cmd_mailbox *mailbox;
6128	int ret;
6129
6130	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6131	if (IS_ERR(mailbox))
6132		return PTR_ERR(mailbox);
6133
6134	context = mailbox->buf;
6135	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
6136				HNS_ROCE_CMD_QUERY_CQC, cqn);
6137	if (ret) {
6138		ibdev_err_ratelimited(&hr_dev->ib_dev,
6139				      "failed to process cmd when querying CQ, ret = %d.\n",
6140				      ret);
6141		goto err_mailbox;
6142	}
6143
6144	memcpy(buffer, context, sizeof(*context));
6145
6146err_mailbox:
6147	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6148
6149	return ret;
6150}
6151
6152static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key,
6153				 void *buffer)
6154{
6155	struct hns_roce_v2_mpt_entry *context;
6156	struct hns_roce_cmd_mailbox *mailbox;
6157	int ret;
6158
6159	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6160	if (IS_ERR(mailbox))
6161		return PTR_ERR(mailbox);
6162
6163	context = mailbox->buf;
6164	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT,
6165				key_to_hw_index(key));
6166	if (ret) {
6167		ibdev_err(&hr_dev->ib_dev,
6168			  "failed to process cmd when querying MPT, ret = %d.\n",
6169			  ret);
6170		goto err_mailbox;
6171	}
6172
6173	memcpy(buffer, context, sizeof(*context));
6174
6175err_mailbox:
6176	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6177
6178	return ret;
6179}
6180
6181static void dump_aeqe_log(struct hns_roce_work *irq_work)
6182{
6183	struct hns_roce_dev *hr_dev = irq_work->hr_dev;
6184	struct ib_device *ibdev = &hr_dev->ib_dev;
6185
6186	switch (irq_work->event_type) {
6187	case HNS_ROCE_EVENT_TYPE_PATH_MIG:
6188		ibdev_info(ibdev, "path migrated succeeded.\n");
6189		break;
6190	case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
6191		ibdev_warn(ibdev, "path migration failed.\n");
6192		break;
6193	case HNS_ROCE_EVENT_TYPE_COMM_EST:
6194		break;
6195	case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
6196		ibdev_dbg(ibdev, "send queue drained.\n");
6197		break;
6198	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
6199		ibdev_err(ibdev, "local work queue 0x%x catast error, sub_event type is: %d\n",
6200			  irq_work->queue_num, irq_work->sub_type);
6201		break;
6202	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
6203		ibdev_err(ibdev, "invalid request local work queue 0x%x error.\n",
6204			  irq_work->queue_num);
6205		break;
6206	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
6207		ibdev_err(ibdev, "local access violation work queue 0x%x error, sub_event type is: %d\n",
6208			  irq_work->queue_num, irq_work->sub_type);
6209		break;
6210	case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
6211		ibdev_dbg(ibdev, "SRQ limit reach.\n");
6212		break;
6213	case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
6214		ibdev_dbg(ibdev, "SRQ last wqe reach.\n");
6215		break;
6216	case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
6217		ibdev_err(ibdev, "SRQ catas error.\n");
6218		break;
6219	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
6220		ibdev_err(ibdev, "CQ 0x%x access err.\n", irq_work->queue_num);
6221		break;
6222	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
6223		ibdev_warn(ibdev, "CQ 0x%x overflow\n", irq_work->queue_num);
6224		break;
6225	case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
6226		ibdev_warn(ibdev, "DB overflow.\n");
6227		break;
6228	case HNS_ROCE_EVENT_TYPE_MB:
6229		break;
6230	case HNS_ROCE_EVENT_TYPE_FLR:
6231		ibdev_warn(ibdev, "function level reset.\n");
6232		break;
6233	case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
6234		ibdev_err(ibdev, "xrc domain violation error.\n");
6235		break;
6236	case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
6237		ibdev_err(ibdev, "invalid xrceth error.\n");
6238		break;
6239	default:
6240		ibdev_info(ibdev, "Undefined event %d.\n",
6241			   irq_work->event_type);
6242		break;
6243	}
6244}
6245
6246static void hns_roce_irq_work_handle(struct work_struct *work)
6247{
6248	struct hns_roce_work *irq_work =
6249				container_of(work, struct hns_roce_work, work);
6250	struct hns_roce_dev *hr_dev = irq_work->hr_dev;
6251	int event_type = irq_work->event_type;
6252	u32 queue_num = irq_work->queue_num;
6253
6254	switch (event_type) {
6255	case HNS_ROCE_EVENT_TYPE_PATH_MIG:
6256	case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
6257	case HNS_ROCE_EVENT_TYPE_COMM_EST:
6258	case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
6259	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
6260	case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
6261	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
6262	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
6263	case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
6264	case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
6265		hns_roce_qp_event(hr_dev, queue_num, event_type);
6266		break;
6267	case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
6268	case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
6269		hns_roce_srq_event(hr_dev, queue_num, event_type);
6270		break;
6271	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
6272	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
6273		hns_roce_cq_event(hr_dev, queue_num, event_type);
6274		break;
6275	default:
6276		break;
6277	}
6278
6279	dump_aeqe_log(irq_work);
6280
6281	kfree(irq_work);
6282}
6283
6284static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
6285				      struct hns_roce_eq *eq, u32 queue_num)
6286{
6287	struct hns_roce_work *irq_work;
6288
6289	irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);
6290	if (!irq_work)
6291		return;
6292
6293	INIT_WORK(&irq_work->work, hns_roce_irq_work_handle);
6294	irq_work->hr_dev = hr_dev;
6295	irq_work->event_type = eq->event_type;
6296	irq_work->sub_type = eq->sub_type;
6297	irq_work->queue_num = queue_num;
6298	queue_work(hr_dev->irq_workq, &irq_work->work);
6299}
6300
6301static void update_eq_db(struct hns_roce_eq *eq)
6302{
6303	struct hns_roce_dev *hr_dev = eq->hr_dev;
6304	struct hns_roce_v2_db eq_db = {};
6305
6306	if (eq->type_flag == HNS_ROCE_AEQ) {
6307		hr_reg_write(&eq_db, EQ_DB_CMD,
6308			     eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
6309			     HNS_ROCE_EQ_DB_CMD_AEQ :
6310			     HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
6311	} else {
6312		hr_reg_write(&eq_db, EQ_DB_TAG, eq->eqn);
6313
6314		hr_reg_write(&eq_db, EQ_DB_CMD,
6315			     eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
6316			     HNS_ROCE_EQ_DB_CMD_CEQ :
6317			     HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
6318	}
6319
6320	hr_reg_write(&eq_db, EQ_DB_CI, eq->cons_index);
6321
6322	hns_roce_write64(hr_dev, (__le32 *)&eq_db, eq->db_reg);
6323}
6324
6325static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
6326{
6327	struct hns_roce_aeqe *aeqe;
6328
6329	aeqe = hns_roce_buf_offset(eq->mtr.kmem,
6330				   (eq->cons_index & (eq->entries - 1)) *
6331				   eq->eqe_size);
6332
6333	return (hr_reg_read(aeqe, AEQE_OWNER) ^
6334		!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
6335}
6336
6337static irqreturn_t hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
6338				       struct hns_roce_eq *eq)
6339{
6340	struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);
6341	irqreturn_t aeqe_found = IRQ_NONE;
6342	int num_aeqes = 0;
6343	int event_type;
6344	u32 queue_num;
6345	int sub_type;
6346
6347	while (aeqe && num_aeqes < HNS_AEQ_POLLING_BUDGET) {
6348		/* Make sure we read AEQ entry after we have checked the
6349		 * ownership bit
6350		 */
6351		dma_rmb();
6352
6353		event_type = hr_reg_read(aeqe, AEQE_EVENT_TYPE);
6354		sub_type = hr_reg_read(aeqe, AEQE_SUB_TYPE);
6355		queue_num = hr_reg_read(aeqe, AEQE_EVENT_QUEUE_NUM);
6356
6357		switch (event_type) {
6358		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
6359		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
6360		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
6361		case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
6362		case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
6363			hns_roce_flush_cqe(hr_dev, queue_num);
6364			break;
6365		case HNS_ROCE_EVENT_TYPE_MB:
6366			hns_roce_cmd_event(hr_dev,
6367					le16_to_cpu(aeqe->event.cmd.token),
6368					aeqe->event.cmd.status,
6369					le64_to_cpu(aeqe->event.cmd.out_param));
6370			break;
6371		default:
6372			break;
6373		}
6374
6375		eq->event_type = event_type;
6376		eq->sub_type = sub_type;
6377		++eq->cons_index;
6378		aeqe_found = IRQ_HANDLED;
6379		trace_hns_ae_info(event_type, aeqe, eq->eqe_size);
6380
6381		atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_AEQE_CNT]);
6382
6383		hns_roce_v2_init_irq_work(hr_dev, eq, queue_num);
6384
6385		aeqe = next_aeqe_sw_v2(eq);
6386		++num_aeqes;
6387	}
6388
6389	update_eq_db(eq);
6390
6391	return IRQ_RETVAL(aeqe_found);
6392}
6393
6394static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
6395{
6396	struct hns_roce_ceqe *ceqe;
6397
6398	ceqe = hns_roce_buf_offset(eq->mtr.kmem,
6399				   (eq->cons_index & (eq->entries - 1)) *
6400				   eq->eqe_size);
6401
6402	return (hr_reg_read(ceqe, CEQE_OWNER) ^
6403		!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
6404}
6405
6406static irqreturn_t hns_roce_v2_ceq_int(struct hns_roce_eq *eq)
6407{
6408	queue_work(system_bh_wq, &eq->work);
6409
6410	return IRQ_HANDLED;
6411}
6412
6413static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
6414{
6415	struct hns_roce_eq *eq = eq_ptr;
6416	struct hns_roce_dev *hr_dev = eq->hr_dev;
6417	irqreturn_t int_work;
6418
6419	if (eq->type_flag == HNS_ROCE_CEQ)
6420		/* Completion event interrupt */
6421		int_work = hns_roce_v2_ceq_int(eq);
6422	else
6423		/* Asynchronous event interrupt */
6424		int_work = hns_roce_v2_aeq_int(hr_dev, eq);
6425
6426	return IRQ_RETVAL(int_work);
6427}
6428
6429static irqreturn_t abnormal_interrupt_basic(struct hns_roce_dev *hr_dev,
6430					    u32 int_st)
6431{
6432	struct pci_dev *pdev = hr_dev->pci_dev;
6433	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
6434	const struct hnae3_ae_ops *ops = ae_dev->ops;
6435	enum hnae3_reset_type reset_type;
6436	irqreturn_t int_work = IRQ_NONE;
6437	u32 int_en;
6438
6439	int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
6440
6441	if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
6442		dev_err(hr_dev->dev, "AEQ overflow!\n");
6443
6444		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG,
6445			   1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S);
6446
6447		reset_type = hr_dev->is_vf ?
6448			     HNAE3_VF_FUNC_RESET : HNAE3_FUNC_RESET;
6449
6450		/* Set reset level for reset_event() */
6451		if (ops->set_default_reset_request)
6452			ops->set_default_reset_request(ae_dev, reset_type);
6453		if (ops->reset_event)
6454			ops->reset_event(pdev, NULL);
6455
6456		int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
6457		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
6458
6459		int_work = IRQ_HANDLED;
6460	} else {
6461		dev_err(hr_dev->dev, "there is no basic abn irq found.\n");
6462	}
6463
6464	return IRQ_RETVAL(int_work);
6465}
6466
6467static int fmea_ram_ecc_query(struct hns_roce_dev *hr_dev,
6468			       struct fmea_ram_ecc *ecc_info)
6469{
6470	struct hns_roce_cmq_desc desc;
6471	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
6472	int ret;
6473
6474	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_QUERY_RAM_ECC, true);
6475	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
6476	if (ret)
6477		return ret;
6478
6479	ecc_info->is_ecc_err = hr_reg_read(req, QUERY_RAM_ECC_1BIT_ERR);
6480	ecc_info->res_type = hr_reg_read(req, QUERY_RAM_ECC_RES_TYPE);
6481	ecc_info->index = hr_reg_read(req, QUERY_RAM_ECC_TAG);
6482
6483	return 0;
6484}
6485
6486static int fmea_recover_gmv(struct hns_roce_dev *hr_dev, u32 idx)
6487{
6488	struct hns_roce_cmq_desc desc;
6489	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
6490	u32 addr_upper;
6491	u32 addr_low;
6492	int ret;
6493
6494	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, true);
6495	hr_reg_write(req, CFG_GMV_BT_IDX, idx);
6496
6497	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
6498	if (ret) {
6499		dev_err(hr_dev->dev,
6500			"failed to execute cmd to read gmv, ret = %d.\n", ret);
6501		return ret;
6502	}
6503
6504	addr_low =  hr_reg_read(req, CFG_GMV_BT_BA_L);
6505	addr_upper = hr_reg_read(req, CFG_GMV_BT_BA_H);
6506
6507	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false);
6508	hr_reg_write(req, CFG_GMV_BT_BA_L, addr_low);
6509	hr_reg_write(req, CFG_GMV_BT_BA_H, addr_upper);
6510	hr_reg_write(req, CFG_GMV_BT_IDX, idx);
6511
6512	return hns_roce_cmq_send(hr_dev, &desc, 1);
6513}
6514
6515static u64 fmea_get_ram_res_addr(u32 res_type, __le64 *data)
6516{
6517	if (res_type == ECC_RESOURCE_QPC_TIMER ||
6518	    res_type == ECC_RESOURCE_CQC_TIMER ||
6519	    res_type == ECC_RESOURCE_SCCC)
6520		return le64_to_cpu(*data);
6521
6522	return le64_to_cpu(*data) << HNS_HW_PAGE_SHIFT;
6523}
6524
6525static int fmea_recover_others(struct hns_roce_dev *hr_dev, u32 res_type,
6526			       u32 index)
6527{
6528	u8 write_bt0_op = fmea_ram_res[res_type].write_bt0_op;
6529	u8 read_bt0_op = fmea_ram_res[res_type].read_bt0_op;
6530	struct hns_roce_cmd_mailbox *mailbox;
6531	u64 addr;
6532	int ret;
6533
6534	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6535	if (IS_ERR(mailbox))
6536		return PTR_ERR(mailbox);
6537
6538	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, read_bt0_op, index);
6539	if (ret) {
6540		dev_err(hr_dev->dev,
6541			"failed to execute cmd to read fmea ram, ret = %d.\n",
6542			ret);
6543		goto out;
6544	}
6545
6546	addr = fmea_get_ram_res_addr(res_type, mailbox->buf);
6547
6548	ret = hns_roce_cmd_mbox(hr_dev, addr, 0, write_bt0_op, index);
6549	if (ret)
6550		dev_err(hr_dev->dev,
6551			"failed to execute cmd to write fmea ram, ret = %d.\n",
6552			ret);
6553
6554out:
6555	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6556	return ret;
6557}
6558
6559static void fmea_ram_ecc_recover(struct hns_roce_dev *hr_dev,
6560				 struct fmea_ram_ecc *ecc_info)
6561{
6562	u32 res_type = ecc_info->res_type;
6563	u32 index = ecc_info->index;
6564	int ret;
6565
6566	BUILD_BUG_ON(ARRAY_SIZE(fmea_ram_res) != ECC_RESOURCE_COUNT);
6567
6568	if (res_type >= ECC_RESOURCE_COUNT) {
6569		dev_err(hr_dev->dev, "unsupported fmea ram ecc type %u.\n",
6570			res_type);
6571		return;
6572	}
6573
6574	if (res_type == ECC_RESOURCE_GMV)
6575		ret = fmea_recover_gmv(hr_dev, index);
6576	else
6577		ret = fmea_recover_others(hr_dev, res_type, index);
6578	if (ret)
6579		dev_err(hr_dev->dev,
6580			"failed to recover %s, index = %u, ret = %d.\n",
6581			fmea_ram_res[res_type].name, index, ret);
6582}
6583
6584static void fmea_ram_ecc_work(struct work_struct *ecc_work)
6585{
6586	struct hns_roce_dev *hr_dev =
6587		container_of(ecc_work, struct hns_roce_dev, ecc_work);
6588	struct fmea_ram_ecc ecc_info = {};
6589
6590	if (fmea_ram_ecc_query(hr_dev, &ecc_info)) {
6591		dev_err(hr_dev->dev, "failed to query fmea ram ecc.\n");
6592		return;
6593	}
6594
6595	if (!ecc_info.is_ecc_err) {
6596		dev_err(hr_dev->dev, "there is no fmea ram ecc err found.\n");
6597		return;
6598	}
6599
6600	fmea_ram_ecc_recover(hr_dev, &ecc_info);
6601}
6602
6603static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
6604{
6605	struct hns_roce_dev *hr_dev = dev_id;
6606	irqreturn_t int_work = IRQ_NONE;
6607	u32 int_st;
6608
6609	int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
6610
6611	if (int_st) {
6612		int_work = abnormal_interrupt_basic(hr_dev, int_st);
6613	} else if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
6614		queue_work(hr_dev->irq_workq, &hr_dev->ecc_work);
6615		int_work = IRQ_HANDLED;
6616	} else {
6617		dev_err(hr_dev->dev, "there is no abnormal irq found.\n");
6618	}
6619
6620	return IRQ_RETVAL(int_work);
6621}
6622
6623static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
6624					int eq_num, u32 enable_flag)
6625{
6626	int i;
6627
6628	for (i = 0; i < eq_num; i++)
6629		roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
6630			   i * EQ_REG_OFFSET, enable_flag);
6631
6632	roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, enable_flag);
6633	roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag);
6634}
6635
6636static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
6637{
6638	hns_roce_mtr_destroy(hr_dev, &eq->mtr);
6639}
6640
6641static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev,
6642				    struct hns_roce_eq *eq)
6643{
6644	struct device *dev = hr_dev->dev;
6645	int eqn = eq->eqn;
6646	int ret;
6647	u8 cmd;
6648
6649	if (eqn < hr_dev->caps.num_comp_vectors)
6650		cmd = HNS_ROCE_CMD_DESTROY_CEQC;
6651	else
6652		cmd = HNS_ROCE_CMD_DESTROY_AEQC;
6653
6654	ret = hns_roce_destroy_hw_ctx(hr_dev, cmd, eqn & HNS_ROCE_V2_EQN_M);
6655	if (ret)
6656		dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
6657
6658	free_eq_buf(hr_dev, eq);
6659}
6660
6661static void init_eq_config(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
6662{
6663	eq->db_reg = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
6664	eq->cons_index = 0;
6665	eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
6666	eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
6667	eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
6668	eq->shift = ilog2((unsigned int)eq->entries);
6669}
6670
6671static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,
6672		      void *mb_buf)
6673{
6674	u64 eqe_ba[MTT_MIN_COUNT] = { 0 };
6675	struct hns_roce_eq_context *eqc;
6676	u64 bt_ba = 0;
6677	int ret;
6678
6679	eqc = mb_buf;
6680	memset(eqc, 0, sizeof(struct hns_roce_eq_context));
6681
6682	init_eq_config(hr_dev, eq);
6683
6684	/* if not multi-hop, eqe buffer only use one trunk */
6685	ret = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba,
6686				ARRAY_SIZE(eqe_ba));
6687	if (ret) {
6688		dev_err(hr_dev->dev, "failed to find EQE mtr, ret = %d\n", ret);
6689		return ret;
6690	}
6691
6692	bt_ba = hns_roce_get_mtr_ba(&eq->mtr);
6693
6694	hr_reg_write(eqc, EQC_EQ_ST, HNS_ROCE_V2_EQ_STATE_VALID);
6695	hr_reg_write(eqc, EQC_EQE_HOP_NUM, eq->hop_num);
6696	hr_reg_write(eqc, EQC_OVER_IGNORE, eq->over_ignore);
6697	hr_reg_write(eqc, EQC_COALESCE, eq->coalesce);
6698	hr_reg_write(eqc, EQC_ARM_ST, eq->arm_st);
6699	hr_reg_write(eqc, EQC_EQN, eq->eqn);
6700	hr_reg_write(eqc, EQC_EQE_CNT, HNS_ROCE_EQ_INIT_EQE_CNT);
6701	hr_reg_write(eqc, EQC_EQE_BA_PG_SZ,
6702		     to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift));
6703	hr_reg_write(eqc, EQC_EQE_BUF_PG_SZ,
6704		     to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift));
6705	hr_reg_write(eqc, EQC_EQ_PROD_INDX, HNS_ROCE_EQ_INIT_PROD_IDX);
6706	hr_reg_write(eqc, EQC_EQ_MAX_CNT, eq->eq_max_cnt);
6707
6708	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
6709		if (eq->eq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) {
6710			dev_info(hr_dev->dev, "eq_period(%u) reached the upper limit, adjusted to 65.\n",
6711				 eq->eq_period);
6712			eq->eq_period = HNS_ROCE_MAX_EQ_PERIOD;
6713		}
6714		eq->eq_period *= HNS_ROCE_CLOCK_ADJUST;
6715	}
6716
6717	hr_reg_write(eqc, EQC_EQ_PERIOD, eq->eq_period);
6718	hr_reg_write(eqc, EQC_EQE_REPORT_TIMER, HNS_ROCE_EQ_INIT_REPORT_TIMER);
6719	hr_reg_write(eqc, EQC_EQE_BA_L, bt_ba >> 3);
6720	hr_reg_write(eqc, EQC_EQE_BA_H, bt_ba >> 35);
6721	hr_reg_write(eqc, EQC_SHIFT, eq->shift);
6722	hr_reg_write(eqc, EQC_MSI_INDX, HNS_ROCE_EQ_INIT_MSI_IDX);
6723	hr_reg_write(eqc, EQC_CUR_EQE_BA_L, eqe_ba[0] >> 12);
6724	hr_reg_write(eqc, EQC_CUR_EQE_BA_M, eqe_ba[0] >> 28);
6725	hr_reg_write(eqc, EQC_CUR_EQE_BA_H, eqe_ba[0] >> 60);
6726	hr_reg_write(eqc, EQC_EQ_CONS_INDX, HNS_ROCE_EQ_INIT_CONS_IDX);
6727	hr_reg_write(eqc, EQC_NEX_EQE_BA_L, eqe_ba[1] >> 12);
6728	hr_reg_write(eqc, EQC_NEX_EQE_BA_H, eqe_ba[1] >> 44);
6729	hr_reg_write(eqc, EQC_EQE_SIZE, eq->eqe_size == HNS_ROCE_V3_EQE_SIZE);
6730
6731	return 0;
6732}
6733
6734static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
6735{
6736	struct hns_roce_buf_attr buf_attr = {};
6737	int err;
6738
6739	if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0)
6740		eq->hop_num = 0;
6741	else
6742		eq->hop_num = hr_dev->caps.eqe_hop_num;
6743
6744	buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT;
6745	buf_attr.region[0].size = eq->entries * eq->eqe_size;
6746	buf_attr.region[0].hopnum = eq->hop_num;
6747	buf_attr.region_count = 1;
6748
6749	err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
6750				  hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL,
6751				  0);
6752	if (err)
6753		dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err);
6754
6755	return err;
6756}
6757
6758static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
6759				 struct hns_roce_eq *eq, u8 eq_cmd)
6760{
6761	struct hns_roce_cmd_mailbox *mailbox;
6762	int ret;
6763
6764	/* Allocate mailbox memory */
6765	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6766	if (IS_ERR(mailbox))
6767		return PTR_ERR(mailbox);
6768
6769	ret = alloc_eq_buf(hr_dev, eq);
6770	if (ret)
6771		goto free_cmd_mbox;
6772
6773	ret = config_eqc(hr_dev, eq, mailbox->buf);
6774	if (ret)
6775		goto err_cmd_mbox;
6776
6777	ret = hns_roce_create_hw_ctx(hr_dev, mailbox, eq_cmd, eq->eqn);
6778	if (ret) {
6779		dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n");
6780		goto err_cmd_mbox;
6781	}
6782
6783	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6784
6785	return 0;
6786
6787err_cmd_mbox:
6788	free_eq_buf(hr_dev, eq);
6789
6790free_cmd_mbox:
6791	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6792
6793	return ret;
6794}
6795
6796static void hns_roce_ceq_work(struct work_struct *work)
6797{
6798	struct hns_roce_eq *eq = from_work(eq, work, work);
6799	struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);
6800	struct hns_roce_dev *hr_dev = eq->hr_dev;
6801	int ceqe_num = 0;
6802	u32 cqn;
6803
6804	while (ceqe && ceqe_num < hr_dev->caps.ceqe_depth) {
6805		/* Make sure we read CEQ entry after we have checked the
6806		 * ownership bit
6807		 */
6808		dma_rmb();
6809
6810		cqn = hr_reg_read(ceqe, CEQE_CQN);
6811
6812		hns_roce_cq_completion(hr_dev, cqn);
6813
6814		++eq->cons_index;
6815		++ceqe_num;
6816		atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CEQE_CNT]);
6817
6818		ceqe = next_ceqe_sw_v2(eq);
6819	}
6820
6821	update_eq_db(eq);
6822}
6823
6824static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,
6825				  int comp_num, int aeq_num, int other_num)
6826{
6827	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6828	int i, j;
6829	int ret;
6830
6831	for (i = 0; i < irq_num; i++) {
6832		hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
6833					       GFP_KERNEL);
6834		if (!hr_dev->irq_names[i]) {
6835			ret = -ENOMEM;
6836			goto err_kzalloc_failed;
6837		}
6838	}
6839
6840	/* irq contains: abnormal + AEQ + CEQ */
6841	for (j = 0; j < other_num; j++)
6842		snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6843			 "hns-%s-abn-%d", pci_name(hr_dev->pci_dev), j);
6844
6845	for (j = other_num; j < (other_num + aeq_num); j++)
6846		snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6847			 "hns-%s-aeq-%d", pci_name(hr_dev->pci_dev), j - other_num);
6848
6849	for (j = (other_num + aeq_num); j < irq_num; j++)
6850		snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6851			 "hns-%s-ceq-%d", pci_name(hr_dev->pci_dev),
6852			 j - other_num - aeq_num);
6853
6854	for (j = 0; j < irq_num; j++) {
6855		if (j < other_num) {
6856			ret = request_irq(hr_dev->irq[j],
6857					  hns_roce_v2_msix_interrupt_abn,
6858					  0, hr_dev->irq_names[j], hr_dev);
6859		} else if (j < (other_num + comp_num)) {
6860			INIT_WORK(&eq_table->eq[j - other_num].work,
6861				  hns_roce_ceq_work);
6862			ret = request_irq(eq_table->eq[j - other_num].irq,
6863					  hns_roce_v2_msix_interrupt_eq,
6864					  0, hr_dev->irq_names[j + aeq_num],
6865					  &eq_table->eq[j - other_num]);
6866		} else {
6867			ret = request_irq(eq_table->eq[j - other_num].irq,
6868					  hns_roce_v2_msix_interrupt_eq,
6869					  0, hr_dev->irq_names[j - comp_num],
6870					  &eq_table->eq[j - other_num]);
6871		}
6872
6873		if (ret) {
6874			dev_err(hr_dev->dev, "request irq error!\n");
6875			goto err_request_failed;
6876		}
6877	}
6878
6879	return 0;
6880
6881err_request_failed:
6882	for (j -= 1; j >= 0; j--) {
6883		if (j < other_num) {
6884			free_irq(hr_dev->irq[j], hr_dev);
6885			continue;
6886		}
6887		free_irq(eq_table->eq[j - other_num].irq,
6888			 &eq_table->eq[j - other_num]);
6889		if (j < other_num + comp_num)
6890			cancel_work_sync(&eq_table->eq[j - other_num].work);
6891	}
6892
6893err_kzalloc_failed:
6894	for (i -= 1; i >= 0; i--)
6895		kfree(hr_dev->irq_names[i]);
6896
6897	return ret;
6898}
6899
6900static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)
6901{
6902	int irq_num;
6903	int eq_num;
6904	int i;
6905
6906	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
6907	irq_num = eq_num + hr_dev->caps.num_other_vectors;
6908
6909	for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
6910		free_irq(hr_dev->irq[i], hr_dev);
6911
6912	for (i = 0; i < eq_num; i++) {
6913		free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]);
6914		if (i < hr_dev->caps.num_comp_vectors)
6915			cancel_work_sync(&hr_dev->eq_table.eq[i].work);
6916	}
6917
6918	for (i = 0; i < irq_num; i++)
6919		kfree(hr_dev->irq_names[i]);
6920}
6921
6922static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
6923{
6924	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6925	struct device *dev = hr_dev->dev;
6926	struct hns_roce_eq *eq;
6927	int other_num;
6928	int comp_num;
6929	int aeq_num;
6930	int irq_num;
6931	int eq_num;
6932	u8 eq_cmd;
6933	int ret;
6934	int i;
6935
6936	if (hr_dev->caps.aeqe_depth < HNS_AEQ_POLLING_BUDGET)
6937		return -EINVAL;
6938
6939	other_num = hr_dev->caps.num_other_vectors;
6940	comp_num = hr_dev->caps.num_comp_vectors;
6941	aeq_num = hr_dev->caps.num_aeq_vectors;
6942
6943	eq_num = comp_num + aeq_num;
6944	irq_num = eq_num + other_num;
6945
6946	eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
6947	if (!eq_table->eq)
6948		return -ENOMEM;
6949
6950	/* create eq */
6951	for (i = 0; i < eq_num; i++) {
6952		eq = &eq_table->eq[i];
6953		eq->hr_dev = hr_dev;
6954		eq->eqn = i;
6955		if (i < comp_num) {
6956			/* CEQ */
6957			eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
6958			eq->type_flag = HNS_ROCE_CEQ;
6959			eq->entries = hr_dev->caps.ceqe_depth;
6960			eq->eqe_size = hr_dev->caps.ceqe_size;
6961			eq->irq = hr_dev->irq[i + other_num + aeq_num];
6962			eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
6963			eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
6964		} else {
6965			/* AEQ */
6966			eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
6967			eq->type_flag = HNS_ROCE_AEQ;
6968			eq->entries = hr_dev->caps.aeqe_depth;
6969			eq->eqe_size = hr_dev->caps.aeqe_size;
6970			eq->irq = hr_dev->irq[i - comp_num + other_num];
6971			eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
6972			eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
6973		}
6974
6975		ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
6976		if (ret) {
6977			dev_err(dev, "failed to create eq.\n");
6978			goto err_create_eq_fail;
6979		}
6980	}
6981
6982	INIT_WORK(&hr_dev->ecc_work, fmea_ram_ecc_work);
6983
6984	hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq",
6985						    WQ_MEM_RECLAIM);
6986	if (!hr_dev->irq_workq) {
6987		dev_err(dev, "failed to create irq workqueue.\n");
6988		ret = -ENOMEM;
6989		goto err_create_eq_fail;
6990	}
6991
6992	ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num, aeq_num,
6993				     other_num);
6994	if (ret) {
6995		dev_err(dev, "failed to request irq.\n");
6996		goto err_request_irq_fail;
6997	}
6998
6999	/* enable irq */
7000	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
7001
7002	return 0;
7003
7004err_request_irq_fail:
7005	destroy_workqueue(hr_dev->irq_workq);
7006
7007err_create_eq_fail:
7008	for (i -= 1; i >= 0; i--)
7009		hns_roce_v2_destroy_eqc(hr_dev, &eq_table->eq[i]);
7010	kfree(eq_table->eq);
7011
7012	return ret;
7013}
7014
7015static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
7016{
7017	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
7018	int eq_num;
7019	int i;
7020
7021	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
7022
7023	/* Disable irq */
7024	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
7025
7026	__hns_roce_free_irq(hr_dev);
7027	destroy_workqueue(hr_dev->irq_workq);
7028
7029	for (i = 0; i < eq_num; i++)
7030		hns_roce_v2_destroy_eqc(hr_dev, &eq_table->eq[i]);
7031
7032	kfree(eq_table->eq);
7033}
7034
7035static const struct ib_device_ops hns_roce_v2_dev_ops = {
7036	.destroy_qp = hns_roce_v2_destroy_qp,
7037	.modify_cq = hns_roce_v2_modify_cq,
7038	.poll_cq = hns_roce_v2_poll_cq,
7039	.post_recv = hns_roce_v2_post_recv,
7040	.post_send = hns_roce_v2_post_send,
7041	.query_qp = hns_roce_v2_query_qp,
7042	.req_notify_cq = hns_roce_v2_req_notify_cq,
7043};
7044
7045static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
7046	.modify_srq = hns_roce_v2_modify_srq,
7047	.post_srq_recv = hns_roce_v2_post_srq_recv,
7048	.query_srq = hns_roce_v2_query_srq,
7049};
7050
7051static const struct hns_roce_hw hns_roce_hw_v2 = {
7052	.cmq_init = hns_roce_v2_cmq_init,
7053	.cmq_exit = hns_roce_v2_cmq_exit,
7054	.hw_profile = hns_roce_v2_profile,
7055	.hw_init = hns_roce_v2_init,
7056	.hw_exit = hns_roce_v2_exit,
7057	.post_mbox = v2_post_mbox,
7058	.poll_mbox_done = v2_poll_mbox_done,
7059	.chk_mbox_avail = v2_chk_mbox_is_avail,
7060	.set_gid = hns_roce_v2_set_gid,
7061	.set_mac = hns_roce_v2_set_mac,
7062	.write_mtpt = hns_roce_v2_write_mtpt,
7063	.rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
7064	.frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
7065	.write_cqc = hns_roce_v2_write_cqc,
7066	.set_hem = hns_roce_v2_set_hem,
7067	.clear_hem = hns_roce_v2_clear_hem,
7068	.modify_qp = hns_roce_v2_modify_qp,
7069	.dereg_mr = hns_roce_v2_dereg_mr,
7070	.qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
7071	.init_eq = hns_roce_v2_init_eq_table,
7072	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
7073	.write_srqc = hns_roce_v2_write_srqc,
7074	.query_cqc = hns_roce_v2_query_cqc,
7075	.query_qpc = hns_roce_v2_query_qpc,
7076	.query_mpt = hns_roce_v2_query_mpt,
7077	.query_srqc = hns_roce_v2_query_srqc,
7078	.query_sccc = hns_roce_v2_query_sccc,
7079	.query_hw_counter = hns_roce_hw_v2_query_counter,
7080	.get_dscp = hns_roce_hw_v2_get_dscp,
7081	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
7082	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
7083};
7084
7085static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
7086	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
7087	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
7088	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
7089	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
7090	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
7091	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},
7092	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
7093	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
7094	/* required last entry */
7095	{0, }
7096};
7097
7098MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
7099
7100static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
7101				  struct hnae3_handle *handle)
7102{
7103	struct hns_roce_v2_priv *priv = hr_dev->priv;
7104	const struct pci_device_id *id;
7105	int i;
7106
7107	hr_dev->pci_dev = handle->pdev;
7108	id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev);
7109	hr_dev->is_vf = id->driver_data;
7110	hr_dev->dev = &handle->pdev->dev;
7111	hr_dev->hw = &hns_roce_hw_v2;
7112	hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
7113	hr_dev->odb_offset = hr_dev->sdb_offset;
7114
7115	/* Get info from NIC driver. */
7116	hr_dev->reg_base = handle->rinfo.roce_io_base;
7117	hr_dev->mem_base = handle->rinfo.roce_mem_base;
7118	hr_dev->caps.num_ports = 1;
7119	hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
7120	hr_dev->iboe.phy_port[0] = 0;
7121
7122	addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
7123			    hr_dev->iboe.netdevs[0]->dev_addr);
7124
7125	for (i = 0; i < handle->rinfo.num_vectors; i++)
7126		hr_dev->irq[i] = pci_irq_vector(handle->pdev,
7127						i + handle->rinfo.base_vector);
7128
7129	/* cmd issue mode: 0 is poll, 1 is event */
7130	hr_dev->cmd_mod = 1;
7131	hr_dev->loop_idc = 0;
7132
7133	hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
7134	priv->handle = handle;
7135}
7136
7137static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
7138{
7139	struct hns_roce_dev *hr_dev;
7140	int ret;
7141
7142	hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
7143	if (!hr_dev)
7144		return -ENOMEM;
7145
7146	hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);
7147	if (!hr_dev->priv) {
7148		ret = -ENOMEM;
7149		goto error_failed_kzalloc;
7150	}
7151
7152	hns_roce_hw_v2_get_cfg(hr_dev, handle);
7153
7154	ret = hns_roce_init(hr_dev);
7155	if (ret) {
7156		dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
7157		goto error_failed_roce_init;
7158	}
7159
7160	handle->priv = hr_dev;
7161
7162	return 0;
7163
7164error_failed_roce_init:
7165	kfree(hr_dev->priv);
7166
7167error_failed_kzalloc:
7168	ib_dealloc_device(&hr_dev->ib_dev);
7169
7170	return ret;
7171}
7172
7173static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
7174					   bool reset, bool bond_cleanup)
7175{
7176	struct hns_roce_dev *hr_dev = handle->priv;
7177
7178	if (!hr_dev)
7179		return;
7180
7181	handle->priv = NULL;
7182
7183	hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT;
7184	hns_roce_handle_device_err(hr_dev);
7185
7186	hns_roce_exit(hr_dev, bond_cleanup);
7187	kfree(hr_dev->priv);
7188	ib_dealloc_device(&hr_dev->ib_dev);
7189}
7190
7191static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
7192{
7193	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
7194	const struct pci_device_id *id;
7195	struct device *dev = &handle->pdev->dev;
7196	int ret;
7197
7198	handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
7199
7200	if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) {
7201		handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
7202		goto reset_chk_err;
7203	}
7204
7205	id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
7206	if (!id)
7207		return 0;
7208
7209	if (id->driver_data && handle->pdev->revision == PCI_REVISION_ID_HIP08)
7210		return 0;
7211
7212	ret = __hns_roce_hw_v2_init_instance(handle);
7213	if (ret) {
7214		handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
7215		dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
7216		if (ops->ae_dev_resetting(handle) ||
7217		    ops->get_hw_reset_stat(handle))
7218			goto reset_chk_err;
7219		else
7220			return ret;
7221	}
7222
7223	handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
7224
7225	return 0;
7226
7227reset_chk_err:
7228	dev_err(dev, "Device is busy in resetting state.\n"
7229		     "please retry later.\n");
7230
7231	return -EBUSY;
7232}
7233
7234static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
7235					   bool reset)
7236{
7237	/* Suspend bond to avoid concurrency */
7238	hns_roce_bond_suspend(handle);
7239
7240	if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
7241		goto out;
7242
7243	handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
7244
7245	__hns_roce_hw_v2_uninit_instance(handle, reset, true);
7246
7247	handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
7248
7249out:
7250	hns_roce_bond_resume(handle);
7251}
7252
7253struct hns_roce_dev
7254	*hns_roce_bond_init_client(struct hns_roce_bond_group *bond_grp,
7255				   int func_idx)
7256{
7257	struct hnae3_handle *handle;
7258	int ret;
7259
7260	handle = bond_grp->bond_func_info[func_idx].handle;
7261	if (!handle || !handle->client)
7262		return NULL;
7263
7264	ret = hns_roce_hw_v2_init_instance(handle);
7265	if (ret)
7266		return NULL;
7267
7268	return handle->priv;
7269}
7270
7271void hns_roce_bond_uninit_client(struct hns_roce_bond_group *bond_grp,
7272				 int func_idx)
7273{
7274	struct hnae3_handle *handle = bond_grp->bond_func_info[func_idx].handle;
7275
7276	if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
7277		return;
7278
7279	handle->rinfo.instance_state = HNS_ROCE_STATE_BOND_UNINIT;
7280
7281	__hns_roce_hw_v2_uninit_instance(handle, false, false);
7282
7283	handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
7284}
7285
7286static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
7287{
7288	struct hns_roce_dev *hr_dev;
7289
7290	/* Suspend bond to avoid concurrency */
7291	hns_roce_bond_suspend(handle);
7292
7293	if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
7294		set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
7295		return 0;
7296	}
7297
7298	handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
7299	clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
7300
7301	hr_dev = handle->priv;
7302	if (!hr_dev)
7303		return 0;
7304
7305	hr_dev->active = false;
7306	hr_dev->dis_db = true;
7307
7308	rdma_user_mmap_disassociate(&hr_dev->ib_dev);
7309
7310	hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;
7311
7312	return 0;
7313}
7314
7315static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
7316{
7317	struct device *dev = &handle->pdev->dev;
7318	int ret;
7319
7320	if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
7321			       &handle->rinfo.state)) {
7322		handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
7323		hns_roce_bond_resume(handle);
7324		return 0;
7325	}
7326
7327	handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
7328
7329	dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
7330	ret = __hns_roce_hw_v2_init_instance(handle);
7331	if (ret) {
7332		/* when reset notify type is HNAE3_INIT_CLIENT In reset notify
7333		 * callback function, RoCE Engine reinitialize. If RoCE reinit
7334		 * failed, we should inform NIC driver.
7335		 */
7336		handle->priv = NULL;
7337		dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
7338	} else {
7339		handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
7340		dev_info(dev, "reset done, RoCE client reinit finished.\n");
7341	}
7342
7343	hns_roce_bond_resume(handle);
7344	return ret;
7345}
7346
7347static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
7348{
7349	if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
7350		return 0;
7351
7352	handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
7353	dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
7354	msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);
7355	__hns_roce_hw_v2_uninit_instance(handle, false, false);
7356
7357	return 0;
7358}
7359
7360static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
7361				       enum hnae3_reset_notify_type type)
7362{
7363	int ret = 0;
7364
7365	switch (type) {
7366	case HNAE3_DOWN_CLIENT:
7367		ret = hns_roce_hw_v2_reset_notify_down(handle);
7368		break;
7369	case HNAE3_INIT_CLIENT:
7370		ret = hns_roce_hw_v2_reset_notify_init(handle);
7371		break;
7372	case HNAE3_UNINIT_CLIENT:
7373		ret = hns_roce_hw_v2_reset_notify_uninit(handle);
7374		break;
7375	default:
7376		break;
7377	}
7378
7379	return ret;
7380}
7381
7382static void hns_roce_hw_v2_link_status_change(struct hnae3_handle *handle,
7383					      bool linkup)
7384{
7385	struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv;
7386	struct net_device *netdev = handle->rinfo.netdev;
7387
7388	if (linkup || !hr_dev)
7389		return;
7390
7391	/* For bond device, the link status depends on the upper netdev,
7392	 * and the upper device's link status depends on all the slaves'
7393	 * netdev but not only one. So bond device cannot get a correct
7394	 * link status from this path.
7395	 */
7396	if (hns_roce_get_bond_grp(netdev, get_hr_bus_num(hr_dev)))
7397		return;
7398
7399	ib_dispatch_port_state_event(&hr_dev->ib_dev, netdev);
7400}
7401
7402static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
7403	.init_instance = hns_roce_hw_v2_init_instance,
7404	.uninit_instance = hns_roce_hw_v2_uninit_instance,
7405	.link_status_change = hns_roce_hw_v2_link_status_change,
7406	.reset_notify = hns_roce_hw_v2_reset_notify,
7407};
7408
7409static struct hnae3_client hns_roce_hw_v2_client = {
7410	.name = "hns_roce_hw_v2",
7411	.type = HNAE3_CLIENT_ROCE,
7412	.ops = &hns_roce_hw_v2_ops,
7413};
7414
7415static int __init hns_roce_hw_v2_init(void)
7416{
7417	hns_roce_init_debugfs();
7418	return hnae3_register_client(&hns_roce_hw_v2_client);
7419}
7420
7421static void __exit hns_roce_hw_v2_exit(void)
7422{
7423	hns_roce_dealloc_bond_grp();
7424	hnae3_unregister_client(&hns_roce_hw_v2_client);
7425	hns_roce_cleanup_debugfs();
7426}
7427
7428module_init(hns_roce_hw_v2_init);
7429module_exit(hns_roce_hw_v2_exit);
7430
7431MODULE_LICENSE("Dual BSD/GPL");
7432MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
7433MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
7434MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");
7435MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");