erdma_cq.c

  1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
  2
  3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
  4/*          Kai Shen <kaishen@linux.alibaba.com> */
  5/* Copyright (c) 2020-2022, Alibaba Group. */
  6
  7#include "erdma_verbs.h"
  8
  9static void *get_next_valid_cqe(struct erdma_cq *cq)
 10{
 11	__be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci,
 12				      cq->depth, CQE_SHIFT);
 13	u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
 14			      be32_to_cpu(READ_ONCE(*cqe)));
 15
 16	return owner ^ !!(cq->kern_cq.ci & cq->depth) ? cqe : NULL;
 17}
 18
 19static void notify_cq(struct erdma_cq *cq, u8 solcitied)
 20{
 21	u64 db_data =
 22		FIELD_PREP(ERDMA_CQDB_IDX_MASK, (cq->kern_cq.notify_cnt)) |
 23		FIELD_PREP(ERDMA_CQDB_CQN_MASK, cq->cqn) |
 24		FIELD_PREP(ERDMA_CQDB_ARM_MASK, 1) |
 25		FIELD_PREP(ERDMA_CQDB_SOL_MASK, solcitied) |
 26		FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cq->kern_cq.cmdsn) |
 27		FIELD_PREP(ERDMA_CQDB_CI_MASK, cq->kern_cq.ci);
 28
 29	*cq->kern_cq.dbrec = db_data;
 30	writeq(db_data, cq->kern_cq.db);
 31}
 32
 33int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 34{
 35	struct erdma_cq *cq = to_ecq(ibcq);
 36	unsigned long irq_flags;
 37	int ret = 0;
 38
 39	spin_lock_irqsave(&cq->kern_cq.lock, irq_flags);
 40
 41	notify_cq(cq, (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);
 42
 43	if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && get_next_valid_cqe(cq))
 44		ret = 1;
 45
 46	cq->kern_cq.notify_cnt++;
 47
 48	spin_unlock_irqrestore(&cq->kern_cq.lock, irq_flags);
 49
 50	return ret;
 51}
 52
 53static const enum ib_wc_opcode wc_mapping_table[ERDMA_NUM_OPCODES] = {
 54	[ERDMA_OP_WRITE] = IB_WC_RDMA_WRITE,
 55	[ERDMA_OP_READ] = IB_WC_RDMA_READ,
 56	[ERDMA_OP_SEND] = IB_WC_SEND,
 57	[ERDMA_OP_SEND_WITH_IMM] = IB_WC_SEND,
 58	[ERDMA_OP_RECEIVE] = IB_WC_RECV,
 59	[ERDMA_OP_RECV_IMM] = IB_WC_RECV_RDMA_WITH_IMM,
 60	[ERDMA_OP_RECV_INV] = IB_WC_RECV,
 61	[ERDMA_OP_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,
 62	[ERDMA_OP_RSP_SEND_IMM] = IB_WC_RECV,
 63	[ERDMA_OP_SEND_WITH_INV] = IB_WC_SEND,
 64	[ERDMA_OP_REG_MR] = IB_WC_REG_MR,
 65	[ERDMA_OP_LOCAL_INV] = IB_WC_LOCAL_INV,
 66	[ERDMA_OP_READ_WITH_INV] = IB_WC_RDMA_READ,
 67	[ERDMA_OP_ATOMIC_CAS] = IB_WC_COMP_SWAP,
 68	[ERDMA_OP_ATOMIC_FAA] = IB_WC_FETCH_ADD,
 69};
 70
 71static const struct {
 72	enum erdma_wc_status erdma;
 73	enum ib_wc_status base;
 74	enum erdma_vendor_err vendor;
 75} map_cqe_status[ERDMA_NUM_WC_STATUS] = {
 76	{ ERDMA_WC_SUCCESS, IB_WC_SUCCESS, ERDMA_WC_VENDOR_NO_ERR },
 77	{ ERDMA_WC_GENERAL_ERR, IB_WC_GENERAL_ERR, ERDMA_WC_VENDOR_NO_ERR },
 78	{ ERDMA_WC_RECV_WQE_FORMAT_ERR, IB_WC_GENERAL_ERR,
 79	  ERDMA_WC_VENDOR_INVALID_RQE },
 80	{ ERDMA_WC_RECV_STAG_INVALID_ERR, IB_WC_REM_ACCESS_ERR,
 81	  ERDMA_WC_VENDOR_RQE_INVALID_STAG },
 82	{ ERDMA_WC_RECV_ADDR_VIOLATION_ERR, IB_WC_REM_ACCESS_ERR,
 83	  ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION },
 84	{ ERDMA_WC_RECV_RIGHT_VIOLATION_ERR, IB_WC_REM_ACCESS_ERR,
 85	  ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR },
 86	{ ERDMA_WC_RECV_PDID_ERR, IB_WC_REM_ACCESS_ERR,
 87	  ERDMA_WC_VENDOR_RQE_INVALID_PD },
 88	{ ERDMA_WC_RECV_WARRPING_ERR, IB_WC_REM_ACCESS_ERR,
 89	  ERDMA_WC_VENDOR_RQE_WRAP_ERR },
 90	{ ERDMA_WC_SEND_WQE_FORMAT_ERR, IB_WC_LOC_QP_OP_ERR,
 91	  ERDMA_WC_VENDOR_INVALID_SQE },
 92	{ ERDMA_WC_SEND_WQE_ORD_EXCEED, IB_WC_GENERAL_ERR,
 93	  ERDMA_WC_VENDOR_ZERO_ORD },
 94	{ ERDMA_WC_SEND_STAG_INVALID_ERR, IB_WC_LOC_ACCESS_ERR,
 95	  ERDMA_WC_VENDOR_SQE_INVALID_STAG },
 96	{ ERDMA_WC_SEND_ADDR_VIOLATION_ERR, IB_WC_LOC_ACCESS_ERR,
 97	  ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION },
 98	{ ERDMA_WC_SEND_RIGHT_VIOLATION_ERR, IB_WC_LOC_ACCESS_ERR,
 99	  ERDMA_WC_VENDOR_SQE_ACCESS_ERR },
100	{ ERDMA_WC_SEND_PDID_ERR, IB_WC_LOC_ACCESS_ERR,
101	  ERDMA_WC_VENDOR_SQE_INVALID_PD },
102	{ ERDMA_WC_SEND_WARRPING_ERR, IB_WC_LOC_ACCESS_ERR,
103	  ERDMA_WC_VENDOR_SQE_WARP_ERR },
104	{ ERDMA_WC_FLUSH_ERR, IB_WC_WR_FLUSH_ERR, ERDMA_WC_VENDOR_NO_ERR },
105	{ ERDMA_WC_RETRY_EXC_ERR, IB_WC_RETRY_EXC_ERR, ERDMA_WC_VENDOR_NO_ERR },
106};
107
108static void erdma_process_ud_cqe(struct erdma_cqe *cqe, struct ib_wc *wc)
109{
110	u32 ud_info;
111
112	wc->wc_flags |= (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE);
113	ud_info = be32_to_cpu(cqe->ud.info);
114	wc->network_hdr_type = FIELD_GET(ERDMA_CQE_NTYPE_MASK, ud_info);
115	if (wc->network_hdr_type == ERDMA_NETWORK_TYPE_IPV4)
116		wc->network_hdr_type = RDMA_NETWORK_IPV4;
117	else
118		wc->network_hdr_type = RDMA_NETWORK_IPV6;
119	wc->src_qp = FIELD_GET(ERDMA_CQE_SQPN_MASK, ud_info);
120	wc->sl = FIELD_GET(ERDMA_CQE_SL_MASK, ud_info);
121	wc->pkey_index = 0;
122}
123
124#define ERDMA_POLLCQ_NO_QP 1
125
126static int erdma_poll_one_cqe(struct erdma_cq *cq, struct ib_wc *wc)
127{
128	struct erdma_dev *dev = to_edev(cq->ibcq.device);
129	u8 opcode, syndrome, qtype;
130	struct erdma_kqp *kern_qp;
131	struct erdma_cqe *cqe;
132	struct erdma_qp *qp;
133	u16 wqe_idx, depth;
134	u32 qpn, cqe_hdr;
135	u64 *id_table;
136	u64 *wqe_hdr;
137
138	cqe = get_next_valid_cqe(cq);
139	if (!cqe)
140		return -EAGAIN;
141
142	cq->kern_cq.ci++;
143
144	/* cqbuf should be ready when we poll */
145	dma_rmb();
146
147	qpn = be32_to_cpu(cqe->qpn);
148	wqe_idx = be32_to_cpu(cqe->qe_idx);
149	cqe_hdr = be32_to_cpu(cqe->hdr);
150
151	qp = find_qp_by_qpn(dev, qpn);
152	if (!qp)
153		return ERDMA_POLLCQ_NO_QP;
154
155	kern_qp = &qp->kern_qp;
156
157	qtype = FIELD_GET(ERDMA_CQE_HDR_QTYPE_MASK, cqe_hdr);
158	syndrome = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, cqe_hdr);
159	opcode = FIELD_GET(ERDMA_CQE_HDR_OPCODE_MASK, cqe_hdr);
160
161	if (qtype == ERDMA_CQE_QTYPE_SQ) {
162		id_table = kern_qp->swr_tbl;
163		depth = qp->attrs.sq_size;
164		wqe_hdr = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
165					  qp->attrs.sq_size, SQEBB_SHIFT);
166		kern_qp->sq_ci =
167			FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) +
168			wqe_idx + 1;
169	} else {
170		id_table = kern_qp->rwr_tbl;
171		depth = qp->attrs.rq_size;
172	}
173	wc->wr_id = id_table[wqe_idx & (depth - 1)];
174	wc->byte_len = be32_to_cpu(cqe->size);
175
176	wc->wc_flags = 0;
177
178	wc->opcode = wc_mapping_table[opcode];
179	if (opcode == ERDMA_OP_RECV_IMM || opcode == ERDMA_OP_RSP_SEND_IMM) {
180		wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->imm_data));
181		wc->wc_flags |= IB_WC_WITH_IMM;
182	} else if (opcode == ERDMA_OP_RECV_INV) {
183		wc->ex.invalidate_rkey = be32_to_cpu(cqe->inv_rkey);
184		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
185	}
186
187	if (erdma_device_rocev2(dev) &&
188	    (qp->ibqp.qp_type == IB_QPT_UD || qp->ibqp.qp_type == IB_QPT_GSI))
189		erdma_process_ud_cqe(cqe, wc);
190
191	if (syndrome >= ERDMA_NUM_WC_STATUS)
192		syndrome = ERDMA_WC_GENERAL_ERR;
193
194	wc->status = map_cqe_status[syndrome].base;
195	wc->vendor_err = map_cqe_status[syndrome].vendor;
196	wc->qp = &qp->ibqp;
197
198	return 0;
199}
200
201int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
202{
203	struct erdma_cq *cq = to_ecq(ibcq);
204	unsigned long flags;
205	int npolled, ret;
206
207	spin_lock_irqsave(&cq->kern_cq.lock, flags);
208
209	for (npolled = 0; npolled < num_entries;) {
210		ret = erdma_poll_one_cqe(cq, wc + npolled);
211
212		if (ret == -EAGAIN) /* no received new CQEs. */
213			break;
214		else if (ret) /* ignore invalid CQEs. */
215			continue;
216
217		npolled++;
218	}
219
220	spin_unlock_irqrestore(&cq->kern_cq.lock, flags);
221
222	return npolled;
223}
224
225void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn)
226{
227	struct erdma_cq *cq = to_ecq(ibcq);
228	struct erdma_cqe *cqe, *dst_cqe;
229	u32 prev_cq_ci, cur_cq_ci;
230	u32 ncqe = 0, nqp_cqe = 0;
231	unsigned long flags;
232	u8 owner;
233
234	spin_lock_irqsave(&cq->kern_cq.lock, flags);
235
236	prev_cq_ci = cq->kern_cq.ci;
237
238	while (ncqe < cq->depth && (cqe = get_next_valid_cqe(cq)) != NULL) {
239		++cq->kern_cq.ci;
240		++ncqe;
241	}
242
243	while (ncqe > 0) {
244		cur_cq_ci = prev_cq_ci + ncqe - 1;
245		cqe = get_queue_entry(cq->kern_cq.qbuf, cur_cq_ci, cq->depth,
246				      CQE_SHIFT);
247
248		if (be32_to_cpu(cqe->qpn) == qpn) {
249			++nqp_cqe;
250		} else if (nqp_cqe) {
251			dst_cqe = get_queue_entry(cq->kern_cq.qbuf,
252						  cur_cq_ci + nqp_cqe,
253						  cq->depth, CQE_SHIFT);
254			owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
255					  be32_to_cpu(dst_cqe->hdr));
256			cqe->hdr = cpu_to_be32(
257				(be32_to_cpu(cqe->hdr) &
258				 ~ERDMA_CQE_HDR_OWNER_MASK) |
259				FIELD_PREP(ERDMA_CQE_HDR_OWNER_MASK, owner));
260			memcpy(dst_cqe, cqe, sizeof(*cqe));
261		}
262
263		--ncqe;
264	}
265
266	cq->kern_cq.ci = prev_cq_ci + nqp_cqe;
267	spin_unlock_irqrestore(&cq->kern_cq.lock, flags);
268}