iw_cxgb4.h

   1/*
   2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *	  copyright notice, this list of conditions and the following
  16 *	  disclaimer.
  17 *      - Redistributions in binary form must reproduce the above
  18 *	  copyright notice, this list of conditions and the following
  19 *	  disclaimer in the documentation and/or other materials
  20 *	  provided with the distribution.
  21 *
  22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  29 * SOFTWARE.
  30 */
  31#ifndef __IW_CXGB4_H__
  32#define __IW_CXGB4_H__
  33
  34#include <linux/mutex.h>
  35#include <linux/list.h>
  36#include <linux/spinlock.h>
  37#include <linux/xarray.h>
  38#include <linux/completion.h>
  39#include <linux/netdevice.h>
  40#include <linux/sched/mm.h>
  41#include <linux/pci.h>
  42#include <linux/dma-mapping.h>
  43#include <linux/inet.h>
  44#include <linux/wait.h>
  45#include <linux/kref.h>
  46#include <linux/timer.h>
  47#include <linux/io.h>
  48#include <linux/workqueue.h>
  49
  50#include <asm/byteorder.h>
  51
  52#include <net/net_namespace.h>
  53
  54#include <rdma/ib_verbs.h>
  55#include <rdma/iw_cm.h>
  56#include <rdma/rdma_netlink.h>
  57#include <rdma/iw_portmap.h>
  58#include <rdma/restrack.h>
  59
  60#include "cxgb4.h"
  61#include "cxgb4_uld.h"
  62#include "l2t.h"
  63#include <rdma/cxgb4-abi.h>
  64
  65#define DRV_NAME "iw_cxgb4"
  66#define MOD DRV_NAME ":"
  67
  68#ifdef pr_fmt
  69#undef pr_fmt
  70#endif
  71
  72#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  73
  74#include "t4.h"
  75
  76#define PBL_OFF(rdev_p, a) ((a) - (rdev_p)->lldi.vr->pbl.start)
  77#define RQT_OFF(rdev_p, a) ((a) - (rdev_p)->lldi.vr->rq.start)
  78
  79static inline void *cplhdr(struct sk_buff *skb)
  80{
  81	return skb->data;
  82}
  83
  84#define C4IW_ID_TABLE_F_RANDOM 1       /* Pseudo-randomize the id's returned */
  85#define C4IW_ID_TABLE_F_EMPTY  2       /* Table is initially empty */
  86
  87struct c4iw_id_table {
  88	u32 flags;
  89	u32 start;              /* logical minimal id */
  90	u32 last;               /* hint for find */
  91	u32 max;
  92	spinlock_t lock;
  93	unsigned long *table;
  94};
  95
  96struct c4iw_resource {
  97	struct c4iw_id_table tpt_table;
  98	struct c4iw_id_table qid_table;
  99	struct c4iw_id_table pdid_table;
 100	struct c4iw_id_table srq_table;
 101};
 102
 103struct c4iw_qid_list {
 104	struct list_head entry;
 105	u32 qid;
 106};
 107
 108struct c4iw_dev_ucontext {
 109	struct list_head qpids;
 110	struct list_head cqids;
 111	struct mutex lock;
 112	struct kref kref;
 113};
 114
 115enum c4iw_rdev_flags {
 116	T4_FATAL_ERROR = (1<<0),
 117	T4_STATUS_PAGE_DISABLED = (1<<1),
 118};
 119
 120struct c4iw_stat {
 121	u64 total;
 122	u64 cur;
 123	u64 max;
 124	u64 fail;
 125};
 126
 127struct c4iw_stats {
 128	struct mutex lock;
 129	struct c4iw_stat qid;
 130	struct c4iw_stat pd;
 131	struct c4iw_stat stag;
 132	struct c4iw_stat pbl;
 133	struct c4iw_stat rqt;
 134	struct c4iw_stat srqt;
 135	struct c4iw_stat srq;
 136	struct c4iw_stat ocqp;
 137	u64  db_full;
 138	u64  db_empty;
 139	u64  db_drop;
 140	u64  db_state_transitions;
 141	u64  db_fc_interruptions;
 142	u64  tcam_full;
 143	u64  act_ofld_conn_fails;
 144	u64  pas_ofld_conn_fails;
 145	u64  neg_adv;
 146};
 147
 148struct c4iw_hw_queue {
 149	int t4_eq_status_entries;
 150	int t4_max_eq_size;
 151	int t4_max_iq_size;
 152	int t4_max_rq_size;
 153	int t4_max_sq_size;
 154	int t4_max_qp_depth;
 155	int t4_max_cq_depth;
 156	int t4_stat_len;
 157};
 158
 159struct wr_log_entry {
 160	ktime_t post_host_time;
 161	ktime_t poll_host_time;
 162	u64 post_sge_ts;
 163	u64 cqe_sge_ts;
 164	u64 poll_sge_ts;
 165	u16 qid;
 166	u16 wr_id;
 167	u8 opcode;
 168	u8 valid;
 169};
 170
 171struct c4iw_rdev {
 172	struct c4iw_resource resource;
 173	u32 qpmask;
 174	u32 cqmask;
 175	struct c4iw_dev_ucontext uctx;
 176	struct gen_pool *pbl_pool;
 177	struct gen_pool *rqt_pool;
 178	struct gen_pool *ocqp_pool;
 179	u32 flags;
 180	struct cxgb4_lld_info lldi;
 181	unsigned long bar2_pa;
 182	void __iomem *bar2_kva;
 183	unsigned long oc_mw_pa;
 184	void __iomem *oc_mw_kva;
 185	struct c4iw_stats stats;
 186	struct c4iw_hw_queue hw_queue;
 187	struct t4_dev_status_page *status_page;
 188	atomic_t wr_log_idx;
 189	struct wr_log_entry *wr_log;
 190	int wr_log_size;
 191	struct workqueue_struct *free_workq;
 192	struct completion rqt_compl;
 193	struct completion pbl_compl;
 194	struct kref rqt_kref;
 195	struct kref pbl_kref;
 196};
 197
 198static inline int c4iw_fatal_error(struct c4iw_rdev *rdev)
 199{
 200	return rdev->flags & T4_FATAL_ERROR;
 201}
 202
 203static inline int c4iw_num_stags(struct c4iw_rdev *rdev)
 204{
 205	return (int)(rdev->lldi.vr->stag.size >> 5);
 206}
 207
 208#define C4IW_WR_TO (60*HZ)
 209
 210struct c4iw_wr_wait {
 211	struct completion completion;
 212	int ret;
 213	struct kref kref;
 214};
 215
 216void _c4iw_free_wr_wait(struct kref *kref);
 217
 218static inline void c4iw_put_wr_wait(struct c4iw_wr_wait *wr_waitp)
 219{
 220	pr_debug("wr_wait %p ref before put %u\n", wr_waitp,
 221		 kref_read(&wr_waitp->kref));
 222	WARN_ON(kref_read(&wr_waitp->kref) == 0);
 223	kref_put(&wr_waitp->kref, _c4iw_free_wr_wait);
 224}
 225
 226static inline void c4iw_get_wr_wait(struct c4iw_wr_wait *wr_waitp)
 227{
 228	pr_debug("wr_wait %p ref before get %u\n", wr_waitp,
 229		 kref_read(&wr_waitp->kref));
 230	WARN_ON(kref_read(&wr_waitp->kref) == 0);
 231	kref_get(&wr_waitp->kref);
 232}
 233
 234static inline void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)
 235{
 236	wr_waitp->ret = 0;
 237	init_completion(&wr_waitp->completion);
 238}
 239
 240static inline void _c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret,
 241				 bool deref)
 242{
 243	wr_waitp->ret = ret;
 244	complete(&wr_waitp->completion);
 245	if (deref)
 246		c4iw_put_wr_wait(wr_waitp);
 247}
 248
 249static inline void c4iw_wake_up_noref(struct c4iw_wr_wait *wr_waitp, int ret)
 250{
 251	_c4iw_wake_up(wr_waitp, ret, false);
 252}
 253
 254static inline void c4iw_wake_up_deref(struct c4iw_wr_wait *wr_waitp, int ret)
 255{
 256	_c4iw_wake_up(wr_waitp, ret, true);
 257}
 258
 259static inline int c4iw_wait_for_reply(struct c4iw_rdev *rdev,
 260				 struct c4iw_wr_wait *wr_waitp,
 261				 u32 hwtid, u32 qpid,
 262				 const char *func)
 263{
 264	int ret;
 265
 266	if (c4iw_fatal_error(rdev)) {
 267		wr_waitp->ret = -EIO;
 268		goto out;
 269	}
 270
 271	ret = wait_for_completion_timeout(&wr_waitp->completion, C4IW_WR_TO);
 272	if (!ret) {
 273		pr_err("%s - Device %s not responding (disabling device) - tid %u qpid %u\n",
 274		       func, pci_name(rdev->lldi.pdev), hwtid, qpid);
 275		rdev->flags |= T4_FATAL_ERROR;
 276		wr_waitp->ret = -EIO;
 277		goto out;
 278	}
 279	if (wr_waitp->ret)
 280		pr_debug("%s: FW reply %d tid %u qpid %u\n",
 281			 pci_name(rdev->lldi.pdev), wr_waitp->ret, hwtid, qpid);
 282out:
 283	return wr_waitp->ret;
 284}
 285
 286int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb);
 287
 288static inline int c4iw_ref_send_wait(struct c4iw_rdev *rdev,
 289				     struct sk_buff *skb,
 290				     struct c4iw_wr_wait *wr_waitp,
 291				     u32 hwtid, u32 qpid,
 292				     const char *func)
 293{
 294	int ret;
 295
 296	pr_debug("%s wr_wait %p hwtid %u qpid %u\n", func, wr_waitp, hwtid,
 297		 qpid);
 298	c4iw_get_wr_wait(wr_waitp);
 299	ret = c4iw_ofld_send(rdev, skb);
 300	if (ret) {
 301		c4iw_put_wr_wait(wr_waitp);
 302		return ret;
 303	}
 304	return c4iw_wait_for_reply(rdev, wr_waitp, hwtid, qpid, func);
 305}
 306
 307enum db_state {
 308	NORMAL = 0,
 309	FLOW_CONTROL = 1,
 310	RECOVERY = 2,
 311	STOPPED = 3
 312};
 313
 314struct c4iw_dev {
 315	struct ib_device ibdev;
 316	struct c4iw_rdev rdev;
 317	struct xarray cqs;
 318	struct xarray qps;
 319	struct xarray mrs;
 320	struct mutex db_mutex;
 321	struct dentry *debugfs_root;
 322	enum db_state db_state;
 323	struct xarray hwtids;
 324	struct xarray atids;
 325	struct xarray stids;
 326	struct list_head db_fc_list;
 327	u32 avail_ird;
 328	wait_queue_head_t wait;
 329};
 330
 331struct uld_ctx {
 332	struct list_head entry;
 333	struct cxgb4_lld_info lldi;
 334	struct c4iw_dev *dev;
 335	struct work_struct reg_work;
 336};
 337
 338static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)
 339{
 340	return container_of(ibdev, struct c4iw_dev, ibdev);
 341}
 342
 343static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid)
 344{
 345	return xa_load(&rhp->cqs, cqid);
 346}
 347
 348static inline struct c4iw_qp *get_qhp(struct c4iw_dev *rhp, u32 qpid)
 349{
 350	return xa_load(&rhp->qps, qpid);
 351}
 352
 353extern uint c4iw_max_read_depth;
 354
 355static inline int cur_max_read_depth(struct c4iw_dev *dev)
 356{
 357	return min(dev->rdev.lldi.max_ordird_qp, c4iw_max_read_depth);
 358}
 359
 360struct c4iw_pd {
 361	struct ib_pd ibpd;
 362	u32 pdid;
 363	struct c4iw_dev *rhp;
 364};
 365
 366static inline struct c4iw_pd *to_c4iw_pd(struct ib_pd *ibpd)
 367{
 368	return container_of(ibpd, struct c4iw_pd, ibpd);
 369}
 370
 371struct tpt_attributes {
 372	u64 len;
 373	u64 va_fbo;
 374	enum fw_ri_mem_perms perms;
 375	u32 stag;
 376	u32 pdid;
 377	u32 qpid;
 378	u32 pbl_addr;
 379	u32 pbl_size;
 380	u32 state:1;
 381	u32 type:2;
 382	u32 rsvd:1;
 383	u32 remote_invaliate_disable:1;
 384	u32 zbva:1;
 385	u32 mw_bind_enable:1;
 386	u32 page_size:5;
 387};
 388
 389struct c4iw_mr {
 390	struct ib_mr ibmr;
 391	struct ib_umem *umem;
 392	struct c4iw_dev *rhp;
 393	struct sk_buff *dereg_skb;
 394	u64 kva;
 395	struct tpt_attributes attr;
 396	u64 *mpl;
 397	dma_addr_t mpl_addr;
 398	u32 max_mpl_len;
 399	u32 mpl_len;
 400	struct c4iw_wr_wait *wr_waitp;
 401};
 402
 403static inline struct c4iw_mr *to_c4iw_mr(struct ib_mr *ibmr)
 404{
 405	return container_of(ibmr, struct c4iw_mr, ibmr);
 406}
 407
 408struct c4iw_mw {
 409	struct ib_mw ibmw;
 410	struct c4iw_dev *rhp;
 411	struct sk_buff *dereg_skb;
 412	u64 kva;
 413	struct tpt_attributes attr;
 414	struct c4iw_wr_wait *wr_waitp;
 415};
 416
 417static inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw)
 418{
 419	return container_of(ibmw, struct c4iw_mw, ibmw);
 420}
 421
 422struct c4iw_cq {
 423	struct ib_cq ibcq;
 424	struct c4iw_dev *rhp;
 425	struct sk_buff *destroy_skb;
 426	struct t4_cq cq;
 427	spinlock_t lock;
 428	spinlock_t comp_handler_lock;
 429	refcount_t refcnt;
 430	struct completion cq_rel_comp;
 431	struct c4iw_wr_wait *wr_waitp;
 432};
 433
 434static inline struct c4iw_cq *to_c4iw_cq(struct ib_cq *ibcq)
 435{
 436	return container_of(ibcq, struct c4iw_cq, ibcq);
 437}
 438
 439struct c4iw_mpa_attributes {
 440	u8 initiator;
 441	u8 recv_marker_enabled;
 442	u8 xmit_marker_enabled;
 443	u8 crc_enabled;
 444	u8 enhanced_rdma_conn;
 445	u8 version;
 446	u8 p2p_type;
 447};
 448
 449struct c4iw_qp_attributes {
 450	u32 scq;
 451	u32 rcq;
 452	u32 sq_num_entries;
 453	u32 rq_num_entries;
 454	u32 sq_max_sges;
 455	u32 sq_max_sges_rdma_write;
 456	u32 rq_max_sges;
 457	u32 state;
 458	u8 enable_rdma_read;
 459	u8 enable_rdma_write;
 460	u8 enable_bind;
 461	u8 enable_mmid0_fastreg;
 462	u32 max_ord;
 463	u32 max_ird;
 464	u32 pd;
 465	u32 next_state;
 466	char terminate_buffer[52];
 467	u32 terminate_msg_len;
 468	u8 is_terminate_local;
 469	struct c4iw_mpa_attributes mpa_attr;
 470	struct c4iw_ep *llp_stream_handle;
 471	u8 layer_etype;
 472	u8 ecode;
 473	u16 sq_db_inc;
 474	u16 rq_db_inc;
 475	u8 send_term;
 476};
 477
 478struct c4iw_qp {
 479	struct ib_qp ibqp;
 480	struct list_head db_fc_entry;
 481	struct c4iw_dev *rhp;
 482	struct c4iw_ep *ep;
 483	struct c4iw_qp_attributes attr;
 484	struct t4_wq wq;
 485	spinlock_t lock;
 486	struct mutex mutex;
 487	wait_queue_head_t wait;
 488	int sq_sig_all;
 489	struct c4iw_srq *srq;
 490	struct c4iw_ucontext *ucontext;
 491	struct c4iw_wr_wait *wr_waitp;
 492	struct completion qp_rel_comp;
 493	refcount_t qp_refcnt;
 494};
 495
 496static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)
 497{
 498	return container_of(ibqp, struct c4iw_qp, ibqp);
 499}
 500
 501struct c4iw_srq {
 502	struct ib_srq ibsrq;
 503	struct list_head db_fc_entry;
 504	struct c4iw_dev *rhp;
 505	struct t4_srq wq;
 506	struct sk_buff *destroy_skb;
 507	u32 srq_limit;
 508	u32 pdid;
 509	int idx;
 510	u32 flags;
 511	spinlock_t lock; /* protects srq */
 512	struct c4iw_wr_wait *wr_waitp;
 513	bool armed;
 514};
 515
 516static inline struct c4iw_srq *to_c4iw_srq(struct ib_srq *ibsrq)
 517{
 518	return container_of(ibsrq, struct c4iw_srq, ibsrq);
 519}
 520
 521struct c4iw_ucontext {
 522	struct ib_ucontext ibucontext;
 523	struct c4iw_dev_ucontext uctx;
 524	u32 key;
 525	spinlock_t mmap_lock;
 526	struct list_head mmaps;
 527	bool is_32b_cqe;
 528};
 529
 530static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)
 531{
 532	return container_of(c, struct c4iw_ucontext, ibucontext);
 533}
 534
 535enum {
 536	CXGB4_MMAP_BAR,
 537	CXGB4_MMAP_BAR_WC,
 538	CXGB4_MMAP_CONTIG,
 539	CXGB4_MMAP_NON_CONTIG,
 540};
 541
 542struct c4iw_mm_entry {
 543	struct list_head entry;
 544	u64 addr;
 545	u32 key;
 546	void *vaddr;
 547	dma_addr_t dma_addr;
 548	unsigned len;
 549	u8 mmap_flag;
 550};
 551
 552static inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext,
 553						u32 key, unsigned len)
 554{
 555	struct list_head *pos, *nxt;
 556	struct c4iw_mm_entry *mm;
 557
 558	spin_lock(&ucontext->mmap_lock);
 559	list_for_each_safe(pos, nxt, &ucontext->mmaps) {
 560
 561		mm = list_entry(pos, struct c4iw_mm_entry, entry);
 562		if (mm->key == key && mm->len == len) {
 563			list_del_init(&mm->entry);
 564			spin_unlock(&ucontext->mmap_lock);
 565			pr_debug("key 0x%x addr 0x%llx len %d\n", key,
 566				 (unsigned long long)mm->addr, mm->len);
 567			return mm;
 568		}
 569	}
 570	spin_unlock(&ucontext->mmap_lock);
 571	return NULL;
 572}
 573
 574static inline void insert_flag_to_mmap(struct c4iw_rdev *rdev,
 575				       struct c4iw_mm_entry *mm, u64 addr)
 576{
 577	if (addr >= pci_resource_start(rdev->lldi.pdev, 0) &&
 578	    (addr < (pci_resource_start(rdev->lldi.pdev, 0) +
 579		    pci_resource_len(rdev->lldi.pdev, 0))))
 580		mm->mmap_flag = CXGB4_MMAP_BAR;
 581	else if (addr >= pci_resource_start(rdev->lldi.pdev, 2) &&
 582		 (addr < (pci_resource_start(rdev->lldi.pdev, 2) +
 583			 pci_resource_len(rdev->lldi.pdev, 2)))) {
 584		if (addr >= rdev->oc_mw_pa) {
 585			mm->mmap_flag = CXGB4_MMAP_BAR_WC;
 586		} else {
 587			if (is_t4(rdev->lldi.adapter_type))
 588				mm->mmap_flag = CXGB4_MMAP_BAR;
 589			else
 590				mm->mmap_flag = CXGB4_MMAP_BAR_WC;
 591		}
 592	} else {
 593		if (addr)
 594			mm->mmap_flag = CXGB4_MMAP_CONTIG;
 595		else
 596			mm->mmap_flag = CXGB4_MMAP_NON_CONTIG;
 597	}
 598}
 599
 600static inline void insert_mmap(struct c4iw_ucontext *ucontext,
 601			       struct c4iw_mm_entry *mm)
 602{
 603	spin_lock(&ucontext->mmap_lock);
 604	pr_debug("key 0x%x addr 0x%llx len %d\n",
 605		 mm->key, (unsigned long long)mm->addr, mm->len);
 606	list_add_tail(&mm->entry, &ucontext->mmaps);
 607	spin_unlock(&ucontext->mmap_lock);
 608}
 609
 610enum c4iw_qp_attr_mask {
 611	C4IW_QP_ATTR_NEXT_STATE = 1 << 0,
 612	C4IW_QP_ATTR_SQ_DB = 1<<1,
 613	C4IW_QP_ATTR_RQ_DB = 1<<2,
 614	C4IW_QP_ATTR_ENABLE_RDMA_READ = 1 << 7,
 615	C4IW_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8,
 616	C4IW_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9,
 617	C4IW_QP_ATTR_MAX_ORD = 1 << 11,
 618	C4IW_QP_ATTR_MAX_IRD = 1 << 12,
 619	C4IW_QP_ATTR_LLP_STREAM_HANDLE = 1 << 22,
 620	C4IW_QP_ATTR_STREAM_MSG_BUFFER = 1 << 23,
 621	C4IW_QP_ATTR_MPA_ATTR = 1 << 24,
 622	C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 << 25,
 623	C4IW_QP_ATTR_VALID_MODIFY = (C4IW_QP_ATTR_ENABLE_RDMA_READ |
 624				     C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
 625				     C4IW_QP_ATTR_MAX_ORD |
 626				     C4IW_QP_ATTR_MAX_IRD |
 627				     C4IW_QP_ATTR_LLP_STREAM_HANDLE |
 628				     C4IW_QP_ATTR_STREAM_MSG_BUFFER |
 629				     C4IW_QP_ATTR_MPA_ATTR |
 630				     C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE)
 631};
 632
 633int c4iw_modify_qp(struct c4iw_dev *rhp,
 634				struct c4iw_qp *qhp,
 635				enum c4iw_qp_attr_mask mask,
 636				struct c4iw_qp_attributes *attrs,
 637				int internal);
 638
 639enum c4iw_qp_state {
 640	C4IW_QP_STATE_IDLE,
 641	C4IW_QP_STATE_RTS,
 642	C4IW_QP_STATE_ERROR,
 643	C4IW_QP_STATE_TERMINATE,
 644	C4IW_QP_STATE_CLOSING,
 645	C4IW_QP_STATE_TOT
 646};
 647
 648static inline int c4iw_convert_state(enum ib_qp_state ib_state)
 649{
 650	switch (ib_state) {
 651	case IB_QPS_RESET:
 652	case IB_QPS_INIT:
 653		return C4IW_QP_STATE_IDLE;
 654	case IB_QPS_RTS:
 655		return C4IW_QP_STATE_RTS;
 656	case IB_QPS_SQD:
 657		return C4IW_QP_STATE_CLOSING;
 658	case IB_QPS_SQE:
 659		return C4IW_QP_STATE_TERMINATE;
 660	case IB_QPS_ERR:
 661		return C4IW_QP_STATE_ERROR;
 662	default:
 663		return -1;
 664	}
 665}
 666
 667static inline int to_ib_qp_state(int c4iw_qp_state)
 668{
 669	switch (c4iw_qp_state) {
 670	case C4IW_QP_STATE_IDLE:
 671		return IB_QPS_INIT;
 672	case C4IW_QP_STATE_RTS:
 673		return IB_QPS_RTS;
 674	case C4IW_QP_STATE_CLOSING:
 675		return IB_QPS_SQD;
 676	case C4IW_QP_STATE_TERMINATE:
 677		return IB_QPS_SQE;
 678	case C4IW_QP_STATE_ERROR:
 679		return IB_QPS_ERR;
 680	}
 681	return IB_QPS_ERR;
 682}
 683
 684static inline u32 c4iw_ib_to_tpt_access(int a)
 685{
 686	return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
 687	       (a & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0) |
 688	       (a & IB_ACCESS_LOCAL_WRITE ? FW_RI_MEM_ACCESS_LOCAL_WRITE : 0) |
 689	       FW_RI_MEM_ACCESS_LOCAL_READ;
 690}
 691
 692enum c4iw_mmid_state {
 693	C4IW_STAG_STATE_VALID,
 694	C4IW_STAG_STATE_INVALID
 695};
 696
 697#define C4IW_NODE_DESC "cxgb4 Chelsio Communications"
 698
 699#define MPA_KEY_REQ "MPA ID Req Frame"
 700#define MPA_KEY_REP "MPA ID Rep Frame"
 701
 702#define MPA_MAX_PRIVATE_DATA	256
 703#define MPA_ENHANCED_RDMA_CONN	0x10
 704#define MPA_REJECT		0x20
 705#define MPA_CRC			0x40
 706#define MPA_MARKERS		0x80
 707#define MPA_FLAGS_MASK		0xE0
 708
 709#define MPA_V2_PEER2PEER_MODEL          0x8000
 710#define MPA_V2_ZERO_LEN_FPDU_RTR        0x4000
 711#define MPA_V2_RDMA_WRITE_RTR           0x8000
 712#define MPA_V2_RDMA_READ_RTR            0x4000
 713#define MPA_V2_IRD_ORD_MASK             0x3FFF
 714
 715#define c4iw_put_ep(ep) {						\
 716	pr_debug("put_ep ep %p refcnt %d\n",		\
 717		 ep, kref_read(&((ep)->kref)));				\
 718	WARN_ON(kref_read(&((ep)->kref)) < 1);				\
 719	kref_put(&((ep)->kref), _c4iw_free_ep);				\
 720}
 721
 722#define c4iw_get_ep(ep) {						\
 723	pr_debug("get_ep ep %p, refcnt %d\n",		\
 724		 ep, kref_read(&((ep)->kref)));				\
 725	kref_get(&((ep)->kref));					\
 726}
 727void _c4iw_free_ep(struct kref *kref);
 728
 729struct mpa_message {
 730	u8 key[16];
 731	u8 flags;
 732	u8 revision;
 733	__be16 private_data_size;
 734	u8 private_data[];
 735};
 736
 737struct mpa_v2_conn_params {
 738	__be16 ird;
 739	__be16 ord;
 740};
 741
 742struct terminate_message {
 743	u8 layer_etype;
 744	u8 ecode;
 745	__be16 hdrct_rsvd;
 746	u8 len_hdrs[];
 747};
 748
 749#define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28)
 750
 751enum c4iw_layers_types {
 752	LAYER_RDMAP		= 0x00,
 753	LAYER_DDP		= 0x10,
 754	LAYER_MPA		= 0x20,
 755	RDMAP_LOCAL_CATA	= 0x00,
 756	RDMAP_REMOTE_PROT	= 0x01,
 757	RDMAP_REMOTE_OP		= 0x02,
 758	DDP_LOCAL_CATA		= 0x00,
 759	DDP_TAGGED_ERR		= 0x01,
 760	DDP_UNTAGGED_ERR	= 0x02,
 761	DDP_LLP			= 0x03
 762};
 763
 764enum c4iw_rdma_ecodes {
 765	RDMAP_INV_STAG		= 0x00,
 766	RDMAP_BASE_BOUNDS	= 0x01,
 767	RDMAP_ACC_VIOL		= 0x02,
 768	RDMAP_STAG_NOT_ASSOC	= 0x03,
 769	RDMAP_TO_WRAP		= 0x04,
 770	RDMAP_INV_VERS		= 0x05,
 771	RDMAP_INV_OPCODE	= 0x06,
 772	RDMAP_STREAM_CATA	= 0x07,
 773	RDMAP_GLOBAL_CATA	= 0x08,
 774	RDMAP_CANT_INV_STAG	= 0x09,
 775	RDMAP_UNSPECIFIED	= 0xff
 776};
 777
 778enum c4iw_ddp_ecodes {
 779	DDPT_INV_STAG		= 0x00,
 780	DDPT_BASE_BOUNDS	= 0x01,
 781	DDPT_STAG_NOT_ASSOC	= 0x02,
 782	DDPT_TO_WRAP		= 0x03,
 783	DDPT_INV_VERS		= 0x04,
 784	DDPU_INV_QN		= 0x01,
 785	DDPU_INV_MSN_NOBUF	= 0x02,
 786	DDPU_INV_MSN_RANGE	= 0x03,
 787	DDPU_INV_MO		= 0x04,
 788	DDPU_MSG_TOOBIG		= 0x05,
 789	DDPU_INV_VERS		= 0x06
 790};
 791
 792enum c4iw_mpa_ecodes {
 793	MPA_CRC_ERR		= 0x02,
 794	MPA_MARKER_ERR          = 0x03,
 795	MPA_LOCAL_CATA          = 0x05,
 796	MPA_INSUFF_IRD          = 0x06,
 797	MPA_NOMATCH_RTR         = 0x07,
 798};
 799
 800enum c4iw_ep_state {
 801	IDLE = 0,
 802	LISTEN,
 803	CONNECTING,
 804	MPA_REQ_WAIT,
 805	MPA_REQ_SENT,
 806	MPA_REQ_RCVD,
 807	MPA_REP_SENT,
 808	FPDU_MODE,
 809	ABORTING,
 810	CLOSING,
 811	MORIBUND,
 812	DEAD,
 813};
 814
 815enum c4iw_ep_flags {
 816	PEER_ABORT_IN_PROGRESS	= 0,
 817	ABORT_REQ_IN_PROGRESS	= 1,
 818	RELEASE_RESOURCES	= 2,
 819	CLOSE_SENT		= 3,
 820	TIMEOUT                 = 4,
 821	QP_REFERENCED           = 5,
 822	STOP_MPA_TIMER		= 7,
 823};
 824
 825enum c4iw_ep_history {
 826	ACT_OPEN_REQ            = 0,
 827	ACT_OFLD_CONN           = 1,
 828	ACT_OPEN_RPL            = 2,
 829	ACT_ESTAB               = 3,
 830	PASS_ACCEPT_REQ         = 4,
 831	PASS_ESTAB              = 5,
 832	ABORT_UPCALL            = 6,
 833	ESTAB_UPCALL            = 7,
 834	CLOSE_UPCALL            = 8,
 835	ULP_ACCEPT              = 9,
 836	ULP_REJECT              = 10,
 837	TIMEDOUT                = 11,
 838	PEER_ABORT              = 12,
 839	PEER_CLOSE              = 13,
 840	CONNREQ_UPCALL          = 14,
 841	ABORT_CONN              = 15,
 842	DISCONN_UPCALL          = 16,
 843	EP_DISC_CLOSE           = 17,
 844	EP_DISC_ABORT           = 18,
 845	CONN_RPL_UPCALL         = 19,
 846	ACT_RETRY_NOMEM         = 20,
 847	ACT_RETRY_INUSE         = 21,
 848	CLOSE_CON_RPL		= 22,
 849	EP_DISC_FAIL		= 24,
 850	QP_REFED		= 25,
 851	QP_DEREFED		= 26,
 852	CM_ID_REFED		= 27,
 853	CM_ID_DEREFED		= 28,
 854};
 855
 856enum conn_pre_alloc_buffers {
 857	CN_ABORT_REQ_BUF,
 858	CN_ABORT_RPL_BUF,
 859	CN_CLOSE_CON_REQ_BUF,
 860	CN_DESTROY_BUF,
 861	CN_FLOWC_BUF,
 862	CN_MAX_CON_BUF
 863};
 864
 865enum {
 866	FLOWC_LEN = offsetof(struct fw_flowc_wr, mnemval[FW_FLOWC_MNEM_MAX])
 867};
 868
 869union cpl_wr_size {
 870	struct cpl_abort_req abrt_req;
 871	struct cpl_abort_rpl abrt_rpl;
 872	struct fw_ri_wr ri_req;
 873	struct cpl_close_con_req close_req;
 874	char flowc_buf[FLOWC_LEN];
 875};
 876
 877struct c4iw_ep_common {
 878	struct iw_cm_id *cm_id;
 879	struct c4iw_qp *qp;
 880	struct c4iw_dev *dev;
 881	struct sk_buff_head ep_skb_list;
 882	enum c4iw_ep_state state;
 883	struct kref kref;
 884	struct mutex mutex;
 885	struct sockaddr_storage local_addr;
 886	struct sockaddr_storage remote_addr;
 887	struct c4iw_wr_wait *wr_waitp;
 888	unsigned long flags;
 889	unsigned long history;
 890};
 891
 892struct c4iw_listen_ep {
 893	struct c4iw_ep_common com;
 894	unsigned int stid;
 895	int backlog;
 896};
 897
 898struct c4iw_ep_stats {
 899	unsigned connect_neg_adv;
 900	unsigned abort_neg_adv;
 901};
 902
 903struct c4iw_ep {
 904	struct c4iw_ep_common com;
 905	struct c4iw_ep *parent_ep;
 906	struct timer_list timer;
 907	struct list_head entry;
 908	unsigned int atid;
 909	u32 hwtid;
 910	u32 snd_seq;
 911	u32 rcv_seq;
 912	struct l2t_entry *l2t;
 913	struct dst_entry *dst;
 914	struct sk_buff *mpa_skb;
 915	struct c4iw_mpa_attributes mpa_attr;
 916	u8 mpa_pkt[sizeof(struct mpa_message) + MPA_MAX_PRIVATE_DATA];
 917	unsigned int mpa_pkt_len;
 918	u32 ird;
 919	u32 ord;
 920	u32 smac_idx;
 921	u32 tx_chan;
 922	u32 mtu;
 923	u16 mss;
 924	u16 emss;
 925	u16 plen;
 926	u16 rss_qid;
 927	u16 txq_idx;
 928	u16 ctrlq_idx;
 929	u8 tos;
 930	u8 retry_with_mpa_v1;
 931	u8 tried_with_mpa_v1;
 932	unsigned int retry_count;
 933	int snd_win;
 934	int rcv_win;
 935	u32 snd_wscale;
 936	struct c4iw_ep_stats stats;
 937	u32 srqe_idx;
 938	u32 rx_pdu_out_cnt;
 939	struct sk_buff *peer_abort_skb;
 940};
 941
 942static inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id)
 943{
 944	return cm_id->provider_data;
 945}
 946
 947static inline struct c4iw_listen_ep *to_listen_ep(struct iw_cm_id *cm_id)
 948{
 949	return cm_id->provider_data;
 950}
 951
 952static inline int ocqp_supported(const struct cxgb4_lld_info *infop)
 953{
 954#if defined(__i386__) || defined(__x86_64__) || defined(CONFIG_PPC64)
 955	return infop->vr->ocq.size > 0;
 956#else
 957	return 0;
 958#endif
 959}
 960
 961u32 c4iw_id_alloc(struct c4iw_id_table *alloc);
 962void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj);
 963int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,
 964			u32 reserved, u32 flags);
 965void c4iw_id_table_free(struct c4iw_id_table *alloc);
 966
 967typedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct sk_buff *skb);
 968
 969void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qpid,
 970		   struct c4iw_dev_ucontext *uctx);
 971u32 c4iw_get_resource(struct c4iw_id_table *id_table);
 972void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry);
 973int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt,
 974		       u32 nr_pdid, u32 nr_srqt);
 975int c4iw_pblpool_create(struct c4iw_rdev *rdev);
 976int c4iw_rqtpool_create(struct c4iw_rdev *rdev);
 977int c4iw_ocqp_pool_create(struct c4iw_rdev *rdev);
 978void c4iw_pblpool_destroy(struct c4iw_rdev *rdev);
 979void c4iw_rqtpool_destroy(struct c4iw_rdev *rdev);
 980void c4iw_ocqp_pool_destroy(struct c4iw_rdev *rdev);
 981void c4iw_destroy_resource(struct c4iw_resource *rscp);
 982void c4iw_register_device(struct work_struct *work);
 983void c4iw_unregister_device(struct c4iw_dev *dev);
 984int __init c4iw_cm_init(void);
 985void c4iw_cm_term(void);
 986void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev,
 987			       struct c4iw_dev_ucontext *uctx);
 988void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev,
 989			    struct c4iw_dev_ucontext *uctx);
 990int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 991int c4iw_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 992		   const struct ib_send_wr **bad_wr);
 993int c4iw_post_receive(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
 994		      const struct ib_recv_wr **bad_wr);
 995int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
 996int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog);
 997int c4iw_destroy_listen(struct iw_cm_id *cm_id);
 998int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
 999int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
1000void c4iw_qp_add_ref(struct ib_qp *qp);
1001void c4iw_qp_rem_ref(struct ib_qp *qp);
1002struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1003			    u32 max_num_sg);
1004int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1005		   unsigned int *sg_offset);
1006void c4iw_dealloc(struct uld_ctx *ctx);
1007struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,
1008					   u64 length, u64 virt, int acc,
1009					   struct ib_dmah *dmah,
1010					   struct ib_udata *udata);
1011struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
1012int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
1013int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
1014void c4iw_cq_rem_ref(struct c4iw_cq *chp);
1015int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1016		   struct uverbs_attr_bundle *attrs);
1017int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
1018int c4iw_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *attr,
1019		    enum ib_srq_attr_mask srq_attr_mask,
1020		    struct ib_udata *udata);
1021int c4iw_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata);
1022int c4iw_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *attrs,
1023		    struct ib_udata *udata);
1024int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata);
1025int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,
1026		   struct ib_udata *udata);
1027int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1028				 int attr_mask, struct ib_udata *udata);
1029int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1030		     int attr_mask, struct ib_qp_init_attr *init_attr);
1031struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn);
1032u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size);
1033void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
1034u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size);
1035void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
1036u32 c4iw_ocqp_pool_alloc(struct c4iw_rdev *rdev, int size);
1037void c4iw_ocqp_pool_free(struct c4iw_rdev *rdev, u32 addr, int size);
1038void c4iw_flush_hw_cq(struct c4iw_cq *chp, struct c4iw_qp *flush_qhp);
1039void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
1040int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
1041int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);
1042int c4iw_flush_sq(struct c4iw_qp *qhp);
1043int c4iw_ev_handler(struct c4iw_dev *rnicp, u32 qid);
1044u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
1045void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,
1046		struct c4iw_dev_ucontext *uctx);
1047u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
1048void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,
1049		struct c4iw_dev_ucontext *uctx);
1050void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe);
1051
1052extern struct cxgb4_client t4c_client;
1053extern c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS];
1054void __iomem *c4iw_bar2_addrs(struct c4iw_rdev *rdev, unsigned int qid,
1055			      enum cxgb4_bar2_qtype qtype,
1056			      unsigned int *pbar2_qid, u64 *pbar2_pa);
1057int c4iw_alloc_srq_idx(struct c4iw_rdev *rdev);
1058void c4iw_free_srq_idx(struct c4iw_rdev *rdev, int idx);
1059extern void c4iw_log_wr_stats(struct t4_wq *wq, struct t4_cqe *cqe);
1060extern int c4iw_wr_log;
1061extern int db_fc_threshold;
1062extern int db_coalescing_threshold;
1063extern int use_dsgl;
1064void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey);
1065void c4iw_dispatch_srq_limit_reached_event(struct c4iw_srq *srq);
1066void c4iw_copy_wr_to_srq(struct t4_srq *srq, union t4_recv_wr *wqe, u8 len16);
1067void c4iw_flush_srqidx(struct c4iw_qp *qhp, u32 srqidx);
1068int c4iw_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
1069		       const struct ib_recv_wr **bad_wr);
1070struct c4iw_wr_wait *c4iw_alloc_wr_wait(gfp_t gfp);
1071
1072int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr);
1073int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq);
1074int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp);
1075int c4iw_fill_res_cm_id_entry(struct sk_buff *msg, struct rdma_cm_id *cm_id);
1076
1077#endif