init.c

   1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2/*
   3 * Copyright(c) 2015 - 2020 Intel Corporation.
   4 * Copyright(c) 2021 Cornelis Networks.
   5 */
   6
   7#include <linux/pci.h>
   8#include <linux/netdevice.h>
   9#include <linux/vmalloc.h>
  10#include <linux/delay.h>
  11#include <linux/xarray.h>
  12#include <linux/module.h>
  13#include <linux/printk.h>
  14#include <linux/hrtimer.h>
  15#include <linux/bitmap.h>
  16#include <linux/numa.h>
  17#include <rdma/rdma_vt.h>
  18
  19#include "hfi.h"
  20#include "device.h"
  21#include "common.h"
  22#include "trace.h"
  23#include "mad.h"
  24#include "sdma.h"
  25#include "debugfs.h"
  26#include "verbs.h"
  27#include "aspm.h"
  28#include "affinity.h"
  29#include "vnic.h"
  30#include "exp_rcv.h"
  31#include "netdev.h"
  32
  33#undef pr_fmt
  34#define pr_fmt(fmt) DRIVER_NAME ": " fmt
  35
  36/*
  37 * min buffers we want to have per context, after driver
  38 */
  39#define HFI1_MIN_USER_CTXT_BUFCNT 7
  40
  41#define HFI1_MIN_EAGER_BUFFER_SIZE (4 * 1024) /* 4KB */
  42#define HFI1_MAX_EAGER_BUFFER_SIZE (256 * 1024) /* 256KB */
  43
  44#define NUM_IB_PORTS 1
  45
  46/*
  47 * Number of user receive contexts we are configured to use (to allow for more
  48 * pio buffers per ctxt, etc.)  Zero means use one user context per CPU.
  49 */
  50int num_user_contexts = -1;
  51module_param_named(num_user_contexts, num_user_contexts, int, 0444);
  52MODULE_PARM_DESC(
  53	num_user_contexts, "Set max number of user contexts to use (default: -1 will use the real (non-HT) CPU count)");
  54
  55uint krcvqs[RXE_NUM_DATA_VL];
  56int krcvqsset;
  57module_param_array(krcvqs, uint, &krcvqsset, S_IRUGO);
  58MODULE_PARM_DESC(krcvqs, "Array of the number of non-control kernel receive queues by VL");
  59
  60/* computed based on above array */
  61unsigned long n_krcvqs;
  62
  63static unsigned hfi1_rcvarr_split = 25;
  64module_param_named(rcvarr_split, hfi1_rcvarr_split, uint, S_IRUGO);
  65MODULE_PARM_DESC(rcvarr_split, "Percent of context's RcvArray entries used for Eager buffers");
  66
  67static uint eager_buffer_size = (8 << 20); /* 8MB */
  68module_param(eager_buffer_size, uint, S_IRUGO);
  69MODULE_PARM_DESC(eager_buffer_size, "Size of the eager buffers, default: 8MB");
  70
  71static uint rcvhdrcnt = 2048; /* 2x the max eager buffer count */
  72module_param_named(rcvhdrcnt, rcvhdrcnt, uint, S_IRUGO);
  73MODULE_PARM_DESC(rcvhdrcnt, "Receive header queue count (default 2048)");
  74
  75static uint hfi1_hdrq_entsize = 32;
  76module_param_named(hdrq_entsize, hfi1_hdrq_entsize, uint, 0444);
  77MODULE_PARM_DESC(hdrq_entsize, "Size of header queue entries: 2 - 8B, 16 - 64B, 32 - 128B (default)");
  78
  79unsigned int user_credit_return_threshold = 33;	/* default is 33% */
  80module_param(user_credit_return_threshold, uint, S_IRUGO);
  81MODULE_PARM_DESC(user_credit_return_threshold, "Credit return threshold for user send contexts, return when unreturned credits passes this many blocks (in percent of allocated blocks, 0 is off)");
  82
  83DEFINE_XARRAY_FLAGS(hfi1_dev_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
  84
  85static int hfi1_create_kctxt(struct hfi1_devdata *dd,
  86			     struct hfi1_pportdata *ppd)
  87{
  88	struct hfi1_ctxtdata *rcd;
  89	int ret;
  90
  91	/* Control context has to be always 0 */
  92	BUILD_BUG_ON(HFI1_CTRL_CTXT != 0);
  93
  94	ret = hfi1_create_ctxtdata(ppd, dd->node, &rcd);
  95	if (ret < 0) {
  96		dd_dev_err(dd, "Kernel receive context allocation failed\n");
  97		return ret;
  98	}
  99
 100	/*
 101	 * Set up the kernel context flags here and now because they use
 102	 * default values for all receive side memories.  User contexts will
 103	 * be handled as they are created.
 104	 */
 105	rcd->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
 106		HFI1_CAP_KGET(NODROP_RHQ_FULL) |
 107		HFI1_CAP_KGET(NODROP_EGR_FULL) |
 108		HFI1_CAP_KGET(DMA_RTAIL);
 109
 110	/* Control context must use DMA_RTAIL */
 111	if (rcd->ctxt == HFI1_CTRL_CTXT)
 112		rcd->flags |= HFI1_CAP_DMA_RTAIL;
 113	rcd->fast_handler = get_dma_rtail_setting(rcd) ?
 114				handle_receive_interrupt_dma_rtail :
 115				handle_receive_interrupt_nodma_rtail;
 116
 117	hfi1_set_seq_cnt(rcd, 1);
 118
 119	rcd->sc = sc_alloc(dd, SC_ACK, rcd->rcvhdrqentsize, dd->node);
 120	if (!rcd->sc) {
 121		dd_dev_err(dd, "Kernel send context allocation failed\n");
 122		return -ENOMEM;
 123	}
 124	hfi1_init_ctxt(rcd->sc);
 125
 126	return 0;
 127}
 128
 129/*
 130 * Create the receive context array and one or more kernel contexts
 131 */
 132int hfi1_create_kctxts(struct hfi1_devdata *dd)
 133{
 134	u16 i;
 135	int ret;
 136
 137	dd->rcd = kcalloc_node(dd->num_rcv_contexts, sizeof(*dd->rcd),
 138			       GFP_KERNEL, dd->node);
 139	if (!dd->rcd)
 140		return -ENOMEM;
 141
 142	for (i = 0; i < dd->first_dyn_alloc_ctxt; ++i) {
 143		ret = hfi1_create_kctxt(dd, dd->pport);
 144		if (ret)
 145			goto bail;
 146	}
 147
 148	return 0;
 149bail:
 150	for (i = 0; dd->rcd && i < dd->first_dyn_alloc_ctxt; ++i)
 151		hfi1_free_ctxt(dd->rcd[i]);
 152
 153	/* All the contexts should be freed, free the array */
 154	kfree(dd->rcd);
 155	dd->rcd = NULL;
 156	return ret;
 157}
 158
 159/*
 160 * Helper routines for the receive context reference count (rcd and uctxt).
 161 */
 162static void hfi1_rcd_init(struct hfi1_ctxtdata *rcd)
 163{
 164	kref_init(&rcd->kref);
 165}
 166
 167/**
 168 * hfi1_rcd_free - When reference is zero clean up.
 169 * @kref: pointer to an initialized rcd data structure
 170 *
 171 */
 172static void hfi1_rcd_free(struct kref *kref)
 173{
 174	unsigned long flags;
 175	struct hfi1_ctxtdata *rcd =
 176		container_of(kref, struct hfi1_ctxtdata, kref);
 177
 178	spin_lock_irqsave(&rcd->dd->uctxt_lock, flags);
 179	rcd->dd->rcd[rcd->ctxt] = NULL;
 180	spin_unlock_irqrestore(&rcd->dd->uctxt_lock, flags);
 181
 182	hfi1_free_ctxtdata(rcd->dd, rcd);
 183
 184	kfree(rcd);
 185}
 186
 187/**
 188 * hfi1_rcd_put - decrement reference for rcd
 189 * @rcd: pointer to an initialized rcd data structure
 190 *
 191 * Use this to put a reference after the init.
 192 */
 193int hfi1_rcd_put(struct hfi1_ctxtdata *rcd)
 194{
 195	if (rcd)
 196		return kref_put(&rcd->kref, hfi1_rcd_free);
 197
 198	return 0;
 199}
 200
 201/**
 202 * hfi1_rcd_get - increment reference for rcd
 203 * @rcd: pointer to an initialized rcd data structure
 204 *
 205 * Use this to get a reference after the init.
 206 *
 207 * Return : reflect kref_get_unless_zero(), which returns non-zero on
 208 * increment, otherwise 0.
 209 */
 210int hfi1_rcd_get(struct hfi1_ctxtdata *rcd)
 211{
 212	return kref_get_unless_zero(&rcd->kref);
 213}
 214
 215/**
 216 * allocate_rcd_index - allocate an rcd index from the rcd array
 217 * @dd: pointer to a valid devdata structure
 218 * @rcd: rcd data structure to assign
 219 * @index: pointer to index that is allocated
 220 *
 221 * Find an empty index in the rcd array, and assign the given rcd to it.
 222 * If the array is full, we are EBUSY.
 223 *
 224 */
 225static int allocate_rcd_index(struct hfi1_devdata *dd,
 226			      struct hfi1_ctxtdata *rcd, u16 *index)
 227{
 228	unsigned long flags;
 229	u16 ctxt;
 230
 231	spin_lock_irqsave(&dd->uctxt_lock, flags);
 232	for (ctxt = 0; ctxt < dd->num_rcv_contexts; ctxt++)
 233		if (!dd->rcd[ctxt])
 234			break;
 235
 236	if (ctxt < dd->num_rcv_contexts) {
 237		rcd->ctxt = ctxt;
 238		dd->rcd[ctxt] = rcd;
 239		hfi1_rcd_init(rcd);
 240	}
 241	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
 242
 243	if (ctxt >= dd->num_rcv_contexts)
 244		return -EBUSY;
 245
 246	*index = ctxt;
 247
 248	return 0;
 249}
 250
 251/**
 252 * hfi1_rcd_get_by_index_safe - validate the ctxt index before accessing the
 253 * array
 254 * @dd: pointer to a valid devdata structure
 255 * @ctxt: the index of an possilbe rcd
 256 *
 257 * This is a wrapper for hfi1_rcd_get_by_index() to validate that the given
 258 * ctxt index is valid.
 259 *
 260 * The caller is responsible for making the _put().
 261 *
 262 */
 263struct hfi1_ctxtdata *hfi1_rcd_get_by_index_safe(struct hfi1_devdata *dd,
 264						 u16 ctxt)
 265{
 266	if (ctxt < dd->num_rcv_contexts)
 267		return hfi1_rcd_get_by_index(dd, ctxt);
 268
 269	return NULL;
 270}
 271
 272/**
 273 * hfi1_rcd_get_by_index - get by index
 274 * @dd: pointer to a valid devdata structure
 275 * @ctxt: the index of an possilbe rcd
 276 *
 277 * We need to protect access to the rcd array.  If access is needed to
 278 * one or more index, get the protecting spinlock and then increment the
 279 * kref.
 280 *
 281 * The caller is responsible for making the _put().
 282 *
 283 */
 284struct hfi1_ctxtdata *hfi1_rcd_get_by_index(struct hfi1_devdata *dd, u16 ctxt)
 285{
 286	unsigned long flags;
 287	struct hfi1_ctxtdata *rcd = NULL;
 288
 289	spin_lock_irqsave(&dd->uctxt_lock, flags);
 290	if (dd->rcd[ctxt]) {
 291		rcd = dd->rcd[ctxt];
 292		if (!hfi1_rcd_get(rcd))
 293			rcd = NULL;
 294	}
 295	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
 296
 297	return rcd;
 298}
 299
 300/*
 301 * Common code for user and kernel context create and setup.
 302 * NOTE: the initial kref is done here (hf1_rcd_init()).
 303 */
 304int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
 305			 struct hfi1_ctxtdata **context)
 306{
 307	struct hfi1_devdata *dd = ppd->dd;
 308	struct hfi1_ctxtdata *rcd;
 309	unsigned kctxt_ngroups = 0;
 310	u32 base;
 311
 312	if (dd->rcv_entries.nctxt_extra >
 313	    dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt)
 314		kctxt_ngroups = (dd->rcv_entries.nctxt_extra -
 315			 (dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt));
 316	rcd = kzalloc_node(sizeof(*rcd), GFP_KERNEL, numa);
 317	if (rcd) {
 318		u32 rcvtids, max_entries;
 319		u16 ctxt;
 320		int ret;
 321
 322		ret = allocate_rcd_index(dd, rcd, &ctxt);
 323		if (ret) {
 324			*context = NULL;
 325			kfree(rcd);
 326			return ret;
 327		}
 328
 329		INIT_LIST_HEAD(&rcd->qp_wait_list);
 330		hfi1_exp_tid_group_init(rcd);
 331		rcd->ppd = ppd;
 332		rcd->dd = dd;
 333		rcd->numa_id = numa;
 334		rcd->rcv_array_groups = dd->rcv_entries.ngroups;
 335		rcd->rhf_rcv_function_map = normal_rhf_rcv_functions;
 336		rcd->slow_handler = handle_receive_interrupt;
 337		rcd->do_interrupt = rcd->slow_handler;
 338		rcd->msix_intr = CCE_NUM_MSIX_VECTORS;
 339
 340		mutex_init(&rcd->exp_mutex);
 341		spin_lock_init(&rcd->exp_lock);
 342		INIT_LIST_HEAD(&rcd->flow_queue.queue_head);
 343		INIT_LIST_HEAD(&rcd->rarr_queue.queue_head);
 344
 345		hfi1_cdbg(PROC, "setting up context %u", rcd->ctxt);
 346
 347		/*
 348		 * Calculate the context's RcvArray entry starting point.
 349		 * We do this here because we have to take into account all
 350		 * the RcvArray entries that previous context would have
 351		 * taken and we have to account for any extra groups assigned
 352		 * to the static (kernel) or dynamic (vnic/user) contexts.
 353		 */
 354		if (ctxt < dd->first_dyn_alloc_ctxt) {
 355			if (ctxt < kctxt_ngroups) {
 356				base = ctxt * (dd->rcv_entries.ngroups + 1);
 357				rcd->rcv_array_groups++;
 358			} else {
 359				base = kctxt_ngroups +
 360					(ctxt * dd->rcv_entries.ngroups);
 361			}
 362		} else {
 363			u16 ct = ctxt - dd->first_dyn_alloc_ctxt;
 364
 365			base = ((dd->n_krcv_queues * dd->rcv_entries.ngroups) +
 366				kctxt_ngroups);
 367			if (ct < dd->rcv_entries.nctxt_extra) {
 368				base += ct * (dd->rcv_entries.ngroups + 1);
 369				rcd->rcv_array_groups++;
 370			} else {
 371				base += dd->rcv_entries.nctxt_extra +
 372					(ct * dd->rcv_entries.ngroups);
 373			}
 374		}
 375		rcd->eager_base = base * dd->rcv_entries.group_size;
 376
 377		rcd->rcvhdrq_cnt = rcvhdrcnt;
 378		rcd->rcvhdrqentsize = hfi1_hdrq_entsize;
 379		rcd->rhf_offset =
 380			rcd->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
 381		/*
 382		 * Simple Eager buffer allocation: we have already pre-allocated
 383		 * the number of RcvArray entry groups. Each ctxtdata structure
 384		 * holds the number of groups for that context.
 385		 *
 386		 * To follow CSR requirements and maintain cacheline alignment,
 387		 * make sure all sizes and bases are multiples of group_size.
 388		 *
 389		 * The expected entry count is what is left after assigning
 390		 * eager.
 391		 */
 392		max_entries = rcd->rcv_array_groups *
 393			dd->rcv_entries.group_size;
 394		rcvtids = ((max_entries * hfi1_rcvarr_split) / 100);
 395		rcd->egrbufs.count = round_down(rcvtids,
 396						dd->rcv_entries.group_size);
 397		if (rcd->egrbufs.count > MAX_EAGER_ENTRIES) {
 398			dd_dev_err(dd, "ctxt%u: requested too many RcvArray entries.\n",
 399				   rcd->ctxt);
 400			rcd->egrbufs.count = MAX_EAGER_ENTRIES;
 401		}
 402		hfi1_cdbg(PROC,
 403			  "ctxt%u: max Eager buffer RcvArray entries: %u",
 404			  rcd->ctxt, rcd->egrbufs.count);
 405
 406		/*
 407		 * Allocate array that will hold the eager buffer accounting
 408		 * data.
 409		 * This will allocate the maximum possible buffer count based
 410		 * on the value of the RcvArray split parameter.
 411		 * The resulting value will be rounded down to the closest
 412		 * multiple of dd->rcv_entries.group_size.
 413		 */
 414		rcd->egrbufs.buffers =
 415			kcalloc_node(rcd->egrbufs.count,
 416				     sizeof(*rcd->egrbufs.buffers),
 417				     GFP_KERNEL, numa);
 418		if (!rcd->egrbufs.buffers)
 419			goto bail;
 420		rcd->egrbufs.rcvtids =
 421			kcalloc_node(rcd->egrbufs.count,
 422				     sizeof(*rcd->egrbufs.rcvtids),
 423				     GFP_KERNEL, numa);
 424		if (!rcd->egrbufs.rcvtids)
 425			goto bail;
 426		rcd->egrbufs.size = eager_buffer_size;
 427		/*
 428		 * The size of the buffers programmed into the RcvArray
 429		 * entries needs to be big enough to handle the highest
 430		 * MTU supported.
 431		 */
 432		if (rcd->egrbufs.size < hfi1_max_mtu) {
 433			rcd->egrbufs.size = __roundup_pow_of_two(hfi1_max_mtu);
 434			hfi1_cdbg(PROC,
 435				  "ctxt%u: eager bufs size too small. Adjusting to %u",
 436				    rcd->ctxt, rcd->egrbufs.size);
 437		}
 438		rcd->egrbufs.rcvtid_size = HFI1_MAX_EAGER_BUFFER_SIZE;
 439
 440		/* Applicable only for statically created kernel contexts */
 441		if (ctxt < dd->first_dyn_alloc_ctxt) {
 442			rcd->opstats = kzalloc_node(sizeof(*rcd->opstats),
 443						    GFP_KERNEL, numa);
 444			if (!rcd->opstats)
 445				goto bail;
 446
 447			/* Initialize TID flow generations for the context */
 448			hfi1_kern_init_ctxt_generations(rcd);
 449		}
 450
 451		*context = rcd;
 452		return 0;
 453	}
 454
 455bail:
 456	*context = NULL;
 457	hfi1_free_ctxt(rcd);
 458	return -ENOMEM;
 459}
 460
 461/**
 462 * hfi1_free_ctxt - free context
 463 * @rcd: pointer to an initialized rcd data structure
 464 *
 465 * This wrapper is the free function that matches hfi1_create_ctxtdata().
 466 * When a context is done being used (kernel or user), this function is called
 467 * for the "final" put to match the kref init from hfi1_create_ctxtdata().
 468 * Other users of the context do a get/put sequence to make sure that the
 469 * structure isn't removed while in use.
 470 */
 471void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd)
 472{
 473	hfi1_rcd_put(rcd);
 474}
 475
 476/*
 477 * Select the largest ccti value over all SLs to determine the intra-
 478 * packet gap for the link.
 479 *
 480 * called with cca_timer_lock held (to protect access to cca_timer
 481 * array), and rcu_read_lock() (to protect access to cc_state).
 482 */
 483void set_link_ipg(struct hfi1_pportdata *ppd)
 484{
 485	struct hfi1_devdata *dd = ppd->dd;
 486	struct cc_state *cc_state;
 487	int i;
 488	u16 cce, ccti_limit, max_ccti = 0;
 489	u16 shift, mult;
 490	u64 src;
 491	u32 current_egress_rate; /* Mbits /sec */
 492	u64 max_pkt_time;
 493	/*
 494	 * max_pkt_time is the maximum packet egress time in units
 495	 * of the fabric clock period 1/(805 MHz).
 496	 */
 497
 498	cc_state = get_cc_state(ppd);
 499
 500	if (!cc_state)
 501		/*
 502		 * This should _never_ happen - rcu_read_lock() is held,
 503		 * and set_link_ipg() should not be called if cc_state
 504		 * is NULL.
 505		 */
 506		return;
 507
 508	for (i = 0; i < OPA_MAX_SLS; i++) {
 509		u16 ccti = ppd->cca_timer[i].ccti;
 510
 511		if (ccti > max_ccti)
 512			max_ccti = ccti;
 513	}
 514
 515	ccti_limit = cc_state->cct.ccti_limit;
 516	if (max_ccti > ccti_limit)
 517		max_ccti = ccti_limit;
 518
 519	cce = cc_state->cct.entries[max_ccti].entry;
 520	shift = (cce & 0xc000) >> 14;
 521	mult = (cce & 0x3fff);
 522
 523	current_egress_rate = active_egress_rate(ppd);
 524
 525	max_pkt_time = egress_cycles(ppd->ibmaxlen, current_egress_rate);
 526
 527	src = (max_pkt_time >> shift) * mult;
 528
 529	src &= SEND_STATIC_RATE_CONTROL_CSR_SRC_RELOAD_SMASK;
 530	src <<= SEND_STATIC_RATE_CONTROL_CSR_SRC_RELOAD_SHIFT;
 531
 532	write_csr(dd, SEND_STATIC_RATE_CONTROL, src);
 533}
 534
 535static enum hrtimer_restart cca_timer_fn(struct hrtimer *t)
 536{
 537	struct cca_timer *cca_timer;
 538	struct hfi1_pportdata *ppd;
 539	int sl;
 540	u16 ccti_timer, ccti_min;
 541	struct cc_state *cc_state;
 542	unsigned long flags;
 543	enum hrtimer_restart ret = HRTIMER_NORESTART;
 544
 545	cca_timer = container_of(t, struct cca_timer, hrtimer);
 546	ppd = cca_timer->ppd;
 547	sl = cca_timer->sl;
 548
 549	rcu_read_lock();
 550
 551	cc_state = get_cc_state(ppd);
 552
 553	if (!cc_state) {
 554		rcu_read_unlock();
 555		return HRTIMER_NORESTART;
 556	}
 557
 558	/*
 559	 * 1) decrement ccti for SL
 560	 * 2) calculate IPG for link (set_link_ipg())
 561	 * 3) restart timer, unless ccti is at min value
 562	 */
 563
 564	ccti_min = cc_state->cong_setting.entries[sl].ccti_min;
 565	ccti_timer = cc_state->cong_setting.entries[sl].ccti_timer;
 566
 567	spin_lock_irqsave(&ppd->cca_timer_lock, flags);
 568
 569	if (cca_timer->ccti > ccti_min) {
 570		cca_timer->ccti--;
 571		set_link_ipg(ppd);
 572	}
 573
 574	if (cca_timer->ccti > ccti_min) {
 575		unsigned long nsec = 1024 * ccti_timer;
 576		/* ccti_timer is in units of 1.024 usec */
 577		hrtimer_forward_now(t, ns_to_ktime(nsec));
 578		ret = HRTIMER_RESTART;
 579	}
 580
 581	spin_unlock_irqrestore(&ppd->cca_timer_lock, flags);
 582	rcu_read_unlock();
 583	return ret;
 584}
 585
 586/*
 587 * Common code for initializing the physical port structure.
 588 */
 589void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,
 590			 struct hfi1_devdata *dd, u8 hw_pidx, u32 port)
 591{
 592	int i;
 593	uint default_pkey_idx;
 594	struct cc_state *cc_state;
 595
 596	ppd->dd = dd;
 597	ppd->hw_pidx = hw_pidx;
 598	ppd->port = port; /* IB port number, not index */
 599	ppd->prev_link_width = LINK_WIDTH_DEFAULT;
 600	/*
 601	 * There are C_VL_COUNT number of PortVLXmitWait counters.
 602	 * Adding 1 to C_VL_COUNT to include the PortXmitWait counter.
 603	 */
 604	for (i = 0; i < C_VL_COUNT + 1; i++) {
 605		ppd->port_vl_xmit_wait_last[i] = 0;
 606		ppd->vl_xmit_flit_cnt[i] = 0;
 607	}
 608
 609	default_pkey_idx = 1;
 610
 611	ppd->pkeys[default_pkey_idx] = DEFAULT_P_KEY;
 612	ppd->part_enforce |= HFI1_PART_ENFORCE_IN;
 613	ppd->pkeys[0] = 0x8001;
 614
 615	INIT_WORK(&ppd->link_vc_work, handle_verify_cap);
 616	INIT_WORK(&ppd->link_up_work, handle_link_up);
 617	INIT_WORK(&ppd->link_down_work, handle_link_down);
 618	INIT_WORK(&ppd->freeze_work, handle_freeze);
 619	INIT_WORK(&ppd->link_downgrade_work, handle_link_downgrade);
 620	INIT_WORK(&ppd->sma_message_work, handle_sma_message);
 621	INIT_WORK(&ppd->link_bounce_work, handle_link_bounce);
 622	INIT_DELAYED_WORK(&ppd->start_link_work, handle_start_link);
 623	INIT_WORK(&ppd->linkstate_active_work, receive_interrupt_work);
 624	INIT_WORK(&ppd->qsfp_info.qsfp_work, qsfp_event);
 625
 626	mutex_init(&ppd->hls_lock);
 627	spin_lock_init(&ppd->qsfp_info.qsfp_lock);
 628
 629	ppd->qsfp_info.ppd = ppd;
 630	ppd->sm_trap_qp = 0x0;
 631	ppd->sa_qp = 0x1;
 632
 633	ppd->hfi1_wq = NULL;
 634
 635	spin_lock_init(&ppd->cca_timer_lock);
 636
 637	for (i = 0; i < OPA_MAX_SLS; i++) {
 638		ppd->cca_timer[i].ppd = ppd;
 639		ppd->cca_timer[i].sl = i;
 640		ppd->cca_timer[i].ccti = 0;
 641		hrtimer_setup(&ppd->cca_timer[i].hrtimer, cca_timer_fn, CLOCK_MONOTONIC,
 642			      HRTIMER_MODE_REL);
 643	}
 644
 645	ppd->cc_max_table_entries = IB_CC_TABLE_CAP_DEFAULT;
 646
 647	spin_lock_init(&ppd->cc_state_lock);
 648	spin_lock_init(&ppd->cc_log_lock);
 649	cc_state = kzalloc(sizeof(*cc_state), GFP_KERNEL);
 650	RCU_INIT_POINTER(ppd->cc_state, cc_state);
 651	if (!cc_state)
 652		goto bail;
 653	return;
 654
 655bail:
 656	dd_dev_err(dd, "Congestion Control Agent disabled for port %d\n", port);
 657}
 658
 659/*
 660 * Do initialization for device that is only needed on
 661 * first detect, not on resets.
 662 */
 663static int loadtime_init(struct hfi1_devdata *dd)
 664{
 665	return 0;
 666}
 667
 668/**
 669 * init_after_reset - re-initialize after a reset
 670 * @dd: the hfi1_ib device
 671 *
 672 * sanity check at least some of the values after reset, and
 673 * ensure no receive or transmit (explicitly, in case reset
 674 * failed
 675 */
 676static int init_after_reset(struct hfi1_devdata *dd)
 677{
 678	int i;
 679	struct hfi1_ctxtdata *rcd;
 680	/*
 681	 * Ensure chip does no sends or receives, tail updates, or
 682	 * pioavail updates while we re-initialize.  This is mostly
 683	 * for the driver data structures, not chip registers.
 684	 */
 685	for (i = 0; i < dd->num_rcv_contexts; i++) {
 686		rcd = hfi1_rcd_get_by_index(dd, i);
 687		hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |
 688			     HFI1_RCVCTRL_INTRAVAIL_DIS |
 689			     HFI1_RCVCTRL_TAILUPD_DIS, rcd);
 690		hfi1_rcd_put(rcd);
 691	}
 692	pio_send_control(dd, PSC_GLOBAL_DISABLE);
 693	for (i = 0; i < dd->num_send_contexts; i++)
 694		sc_disable(dd->send_contexts[i].sc);
 695
 696	return 0;
 697}
 698
 699static void enable_chip(struct hfi1_devdata *dd)
 700{
 701	struct hfi1_ctxtdata *rcd;
 702	u32 rcvmask;
 703	u16 i;
 704
 705	/* enable PIO send */
 706	pio_send_control(dd, PSC_GLOBAL_ENABLE);
 707
 708	/*
 709	 * Enable kernel ctxts' receive and receive interrupt.
 710	 * Other ctxts done as user opens and initializes them.
 711	 */
 712	for (i = 0; i < dd->first_dyn_alloc_ctxt; ++i) {
 713		rcd = hfi1_rcd_get_by_index(dd, i);
 714		if (!rcd)
 715			continue;
 716		rcvmask = HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB;
 717		rcvmask |= HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ?
 718			HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
 719		if (!HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR))
 720			rcvmask |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
 721		if (HFI1_CAP_KGET_MASK(rcd->flags, NODROP_RHQ_FULL))
 722			rcvmask |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
 723		if (HFI1_CAP_KGET_MASK(rcd->flags, NODROP_EGR_FULL))
 724			rcvmask |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
 725		if (HFI1_CAP_IS_KSET(TID_RDMA))
 726			rcvmask |= HFI1_RCVCTRL_TIDFLOW_ENB;
 727		hfi1_rcvctrl(dd, rcvmask, rcd);
 728		sc_enable(rcd->sc);
 729		hfi1_rcd_put(rcd);
 730	}
 731}
 732
 733/**
 734 * create_workqueues - create per port workqueues
 735 * @dd: the hfi1_ib device
 736 */
 737static int create_workqueues(struct hfi1_devdata *dd)
 738{
 739	int pidx;
 740	struct hfi1_pportdata *ppd;
 741
 742	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 743		ppd = dd->pport + pidx;
 744		if (!ppd->hfi1_wq) {
 745			ppd->hfi1_wq =
 746				alloc_workqueue(
 747				    "hfi%d_%d",
 748				    WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM |
 749				    WQ_PERCPU,
 750				    HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES,
 751				    dd->unit, pidx);
 752			if (!ppd->hfi1_wq)
 753				goto wq_error;
 754		}
 755		if (!ppd->link_wq) {
 756			/*
 757			 * Make the link workqueue single-threaded to enforce
 758			 * serialization.
 759			 */
 760			ppd->link_wq =
 761				alloc_workqueue(
 762				    "hfi_link_%d_%d",
 763				    WQ_SYSFS | WQ_MEM_RECLAIM | WQ_UNBOUND,
 764				    1, /* max_active */
 765				    dd->unit, pidx);
 766			if (!ppd->link_wq)
 767				goto wq_error;
 768		}
 769	}
 770	return 0;
 771wq_error:
 772	pr_err("alloc_workqueue failed for port %d\n", pidx + 1);
 773	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 774		ppd = dd->pport + pidx;
 775		if (ppd->hfi1_wq) {
 776			destroy_workqueue(ppd->hfi1_wq);
 777			ppd->hfi1_wq = NULL;
 778		}
 779		if (ppd->link_wq) {
 780			destroy_workqueue(ppd->link_wq);
 781			ppd->link_wq = NULL;
 782		}
 783	}
 784	return -ENOMEM;
 785}
 786
 787/**
 788 * destroy_workqueues - destroy per port workqueues
 789 * @dd: the hfi1_ib device
 790 */
 791static void destroy_workqueues(struct hfi1_devdata *dd)
 792{
 793	int pidx;
 794	struct hfi1_pportdata *ppd;
 795
 796	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 797		ppd = dd->pport + pidx;
 798
 799		if (ppd->hfi1_wq) {
 800			destroy_workqueue(ppd->hfi1_wq);
 801			ppd->hfi1_wq = NULL;
 802		}
 803		if (ppd->link_wq) {
 804			destroy_workqueue(ppd->link_wq);
 805			ppd->link_wq = NULL;
 806		}
 807	}
 808}
 809
 810/**
 811 * enable_general_intr() - Enable the IRQs that will be handled by the
 812 * general interrupt handler.
 813 * @dd: valid devdata
 814 *
 815 */
 816static void enable_general_intr(struct hfi1_devdata *dd)
 817{
 818	set_intr_bits(dd, CCE_ERR_INT, MISC_ERR_INT, true);
 819	set_intr_bits(dd, PIO_ERR_INT, TXE_ERR_INT, true);
 820	set_intr_bits(dd, IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END, true);
 821	set_intr_bits(dd, PBC_INT, GPIO_ASSERT_INT, true);
 822	set_intr_bits(dd, TCRIT_INT, TCRIT_INT, true);
 823	set_intr_bits(dd, IS_DC_START, IS_DC_END, true);
 824	set_intr_bits(dd, IS_SENDCREDIT_START, IS_SENDCREDIT_END, true);
 825}
 826
 827/**
 828 * hfi1_init - do the actual initialization sequence on the chip
 829 * @dd: the hfi1_ib device
 830 * @reinit: re-initializing, so don't allocate new memory
 831 *
 832 * Do the actual initialization sequence on the chip.  This is done
 833 * both from the init routine called from the PCI infrastructure, and
 834 * when we reset the chip, or detect that it was reset internally,
 835 * or it's administratively re-enabled.
 836 *
 837 * Memory allocation here and in called routines is only done in
 838 * the first case (reinit == 0).  We have to be careful, because even
 839 * without memory allocation, we need to re-write all the chip registers
 840 * TIDs, etc. after the reset or enable has completed.
 841 */
 842int hfi1_init(struct hfi1_devdata *dd, int reinit)
 843{
 844	int ret = 0, pidx, lastfail = 0;
 845	unsigned long len;
 846	u16 i;
 847	struct hfi1_ctxtdata *rcd;
 848	struct hfi1_pportdata *ppd;
 849
 850	/* Set up send low level handlers */
 851	dd->process_pio_send = hfi1_verbs_send_pio;
 852	dd->process_dma_send = hfi1_verbs_send_dma;
 853	dd->pio_inline_send = pio_copy;
 854	dd->process_vnic_dma_send = hfi1_vnic_send_dma;
 855
 856	if (is_ax(dd)) {
 857		atomic_set(&dd->drop_packet, DROP_PACKET_ON);
 858		dd->do_drop = true;
 859	} else {
 860		atomic_set(&dd->drop_packet, DROP_PACKET_OFF);
 861		dd->do_drop = false;
 862	}
 863
 864	/* make sure the link is not "up" */
 865	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 866		ppd = dd->pport + pidx;
 867		ppd->linkup = 0;
 868	}
 869
 870	if (reinit)
 871		ret = init_after_reset(dd);
 872	else
 873		ret = loadtime_init(dd);
 874	if (ret)
 875		goto done;
 876
 877	/* dd->rcd can be NULL if early initialization failed */
 878	for (i = 0; dd->rcd && i < dd->first_dyn_alloc_ctxt; ++i) {
 879		/*
 880		 * Set up the (kernel) rcvhdr queue and egr TIDs.  If doing
 881		 * re-init, the simplest way to handle this is to free
 882		 * existing, and re-allocate.
 883		 * Need to re-create rest of ctxt 0 ctxtdata as well.
 884		 */
 885		rcd = hfi1_rcd_get_by_index(dd, i);
 886		if (!rcd)
 887			continue;
 888
 889		lastfail = hfi1_create_rcvhdrq(dd, rcd);
 890		if (!lastfail)
 891			lastfail = hfi1_setup_eagerbufs(rcd);
 892		if (!lastfail)
 893			lastfail = hfi1_kern_exp_rcv_init(rcd, reinit);
 894		if (lastfail) {
 895			dd_dev_err(dd,
 896				   "failed to allocate kernel ctxt's rcvhdrq and/or egr bufs\n");
 897			ret = lastfail;
 898		}
 899		/* enable IRQ */
 900		hfi1_rcd_put(rcd);
 901	}
 902
 903	/* Allocate enough memory for user event notification. */
 904	len = PAGE_ALIGN(chip_rcv_contexts(dd) * HFI1_MAX_SHARED_CTXTS *
 905			 sizeof(*dd->events));
 906	dd->events = vmalloc_user(len);
 907	if (!dd->events)
 908		dd_dev_err(dd, "Failed to allocate user events page\n");
 909	/*
 910	 * Allocate a page for device and port status.
 911	 * Page will be shared amongst all user processes.
 912	 */
 913	dd->status = vmalloc_user(PAGE_SIZE);
 914	if (!dd->status)
 915		dd_dev_err(dd, "Failed to allocate dev status page\n");
 916	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 917		ppd = dd->pport + pidx;
 918		if (dd->status)
 919			/* Currently, we only have one port */
 920			ppd->statusp = &dd->status->port;
 921
 922		set_mtu(ppd);
 923	}
 924
 925	/* enable chip even if we have an error, so we can debug cause */
 926	enable_chip(dd);
 927
 928done:
 929	/*
 930	 * Set status even if port serdes is not initialized
 931	 * so that diags will work.
 932	 */
 933	if (dd->status)
 934		dd->status->dev |= HFI1_STATUS_CHIP_PRESENT |
 935			HFI1_STATUS_INITTED;
 936	if (!ret) {
 937		/* enable all interrupts from the chip */
 938		enable_general_intr(dd);
 939		init_qsfp_int(dd);
 940
 941		/* chip is OK for user apps; mark it as initialized */
 942		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 943			ppd = dd->pport + pidx;
 944
 945			/*
 946			 * start the serdes - must be after interrupts are
 947			 * enabled so we are notified when the link goes up
 948			 */
 949			lastfail = bringup_serdes(ppd);
 950			if (lastfail)
 951				dd_dev_info(dd,
 952					    "Failed to bring up port %u\n",
 953					    ppd->port);
 954
 955			/*
 956			 * Set status even if port serdes is not initialized
 957			 * so that diags will work.
 958			 */
 959			if (ppd->statusp)
 960				*ppd->statusp |= HFI1_STATUS_CHIP_PRESENT |
 961							HFI1_STATUS_INITTED;
 962			if (!ppd->link_speed_enabled)
 963				continue;
 964		}
 965	}
 966
 967	/* if ret is non-zero, we probably should do some cleanup here... */
 968	return ret;
 969}
 970
 971struct hfi1_devdata *hfi1_lookup(int unit)
 972{
 973	return xa_load(&hfi1_dev_table, unit);
 974}
 975
 976/*
 977 * Stop the timers during unit shutdown, or after an error late
 978 * in initialization.
 979 */
 980static void stop_timers(struct hfi1_devdata *dd)
 981{
 982	struct hfi1_pportdata *ppd;
 983	int pidx;
 984
 985	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 986		ppd = dd->pport + pidx;
 987		if (ppd->led_override_timer.function) {
 988			timer_delete_sync(&ppd->led_override_timer);
 989			atomic_set(&ppd->led_override_timer_active, 0);
 990		}
 991	}
 992}
 993
 994/**
 995 * shutdown_device - shut down a device
 996 * @dd: the hfi1_ib device
 997 *
 998 * This is called to make the device quiet when we are about to
 999 * unload the driver, and also when the device is administratively
1000 * disabled.   It does not free any data structures.
1001 * Everything it does has to be setup again by hfi1_init(dd, 1)
1002 */
1003static void shutdown_device(struct hfi1_devdata *dd)
1004{
1005	struct hfi1_pportdata *ppd;
1006	struct hfi1_ctxtdata *rcd;
1007	unsigned pidx;
1008	int i;
1009
1010	if (dd->flags & HFI1_SHUTDOWN)
1011		return;
1012	dd->flags |= HFI1_SHUTDOWN;
1013
1014	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1015		ppd = dd->pport + pidx;
1016
1017		ppd->linkup = 0;
1018		if (ppd->statusp)
1019			*ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
1020					   HFI1_STATUS_IB_READY);
1021	}
1022	dd->flags &= ~HFI1_INITTED;
1023
1024	/* mask and clean up interrupts */
1025	set_intr_bits(dd, IS_FIRST_SOURCE, IS_LAST_SOURCE, false);
1026	msix_clean_up_interrupts(dd);
1027
1028	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1029		for (i = 0; i < dd->num_rcv_contexts; i++) {
1030			rcd = hfi1_rcd_get_by_index(dd, i);
1031			hfi1_rcvctrl(dd, HFI1_RCVCTRL_TAILUPD_DIS |
1032				     HFI1_RCVCTRL_CTXT_DIS |
1033				     HFI1_RCVCTRL_INTRAVAIL_DIS |
1034				     HFI1_RCVCTRL_PKEY_DIS |
1035				     HFI1_RCVCTRL_ONE_PKT_EGR_DIS, rcd);
1036			hfi1_rcd_put(rcd);
1037		}
1038		/*
1039		 * Gracefully stop all sends allowing any in progress to
1040		 * trickle out first.
1041		 */
1042		for (i = 0; i < dd->num_send_contexts; i++)
1043			sc_flush(dd->send_contexts[i].sc);
1044	}
1045
1046	/*
1047	 * Enough for anything that's going to trickle out to have actually
1048	 * done so.
1049	 */
1050	udelay(20);
1051
1052	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1053		ppd = dd->pport + pidx;
1054
1055		/* disable all contexts */
1056		for (i = 0; i < dd->num_send_contexts; i++)
1057			sc_disable(dd->send_contexts[i].sc);
1058		/* disable the send device */
1059		pio_send_control(dd, PSC_GLOBAL_DISABLE);
1060
1061		shutdown_led_override(ppd);
1062
1063		/*
1064		 * Clear SerdesEnable.
1065		 * We can't count on interrupts since we are stopping.
1066		 */
1067		hfi1_quiet_serdes(ppd);
1068		if (ppd->hfi1_wq)
1069			flush_workqueue(ppd->hfi1_wq);
1070		if (ppd->link_wq)
1071			flush_workqueue(ppd->link_wq);
1072	}
1073	sdma_exit(dd);
1074}
1075
1076/**
1077 * hfi1_free_ctxtdata - free a context's allocated data
1078 * @dd: the hfi1_ib device
1079 * @rcd: the ctxtdata structure
1080 *
1081 * free up any allocated data for a context
1082 * It should never change any chip state, or global driver state.
1083 */
1084void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
1085{
1086	u32 e;
1087
1088	if (!rcd)
1089		return;
1090
1091	if (rcd->rcvhdrq) {
1092		dma_free_coherent(&dd->pcidev->dev, rcvhdrq_size(rcd),
1093				  rcd->rcvhdrq, rcd->rcvhdrq_dma);
1094		rcd->rcvhdrq = NULL;
1095		if (hfi1_rcvhdrtail_kvaddr(rcd)) {
1096			dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
1097					  (void *)hfi1_rcvhdrtail_kvaddr(rcd),
1098					  rcd->rcvhdrqtailaddr_dma);
1099			rcd->rcvhdrtail_kvaddr = NULL;
1100		}
1101	}
1102
1103	/* all the RcvArray entries should have been cleared by now */
1104	kfree(rcd->egrbufs.rcvtids);
1105	rcd->egrbufs.rcvtids = NULL;
1106
1107	for (e = 0; e < rcd->egrbufs.alloced; e++) {
1108		if (rcd->egrbufs.buffers[e].addr)
1109			dma_free_coherent(&dd->pcidev->dev,
1110					  rcd->egrbufs.buffers[e].len,
1111					  rcd->egrbufs.buffers[e].addr,
1112					  rcd->egrbufs.buffers[e].dma);
1113	}
1114	kfree(rcd->egrbufs.buffers);
1115	rcd->egrbufs.alloced = 0;
1116	rcd->egrbufs.buffers = NULL;
1117
1118	sc_free(rcd->sc);
1119	rcd->sc = NULL;
1120
1121	vfree(rcd->subctxt_uregbase);
1122	vfree(rcd->subctxt_rcvegrbuf);
1123	vfree(rcd->subctxt_rcvhdr_base);
1124	kfree(rcd->opstats);
1125
1126	rcd->subctxt_uregbase = NULL;
1127	rcd->subctxt_rcvegrbuf = NULL;
1128	rcd->subctxt_rcvhdr_base = NULL;
1129	rcd->opstats = NULL;
1130}
1131
1132/*
1133 * Release our hold on the shared asic data.  If we are the last one,
1134 * return the structure to be finalized outside the lock.  Must be
1135 * holding hfi1_dev_table lock.
1136 */
1137static struct hfi1_asic_data *release_asic_data(struct hfi1_devdata *dd)
1138{
1139	struct hfi1_asic_data *ad;
1140	int other;
1141
1142	if (!dd->asic_data)
1143		return NULL;
1144	dd->asic_data->dds[dd->hfi1_id] = NULL;
1145	other = dd->hfi1_id ? 0 : 1;
1146	ad = dd->asic_data;
1147	dd->asic_data = NULL;
1148	/* return NULL if the other dd still has a link */
1149	return ad->dds[other] ? NULL : ad;
1150}
1151
1152static void finalize_asic_data(struct hfi1_devdata *dd,
1153			       struct hfi1_asic_data *ad)
1154{
1155	clean_up_i2c(dd, ad);
1156	kfree(ad);
1157}
1158
1159/**
1160 * hfi1_free_devdata - cleans up and frees per-unit data structure
1161 * @dd: pointer to a valid devdata structure
1162 *
1163 * It cleans up and frees all data structures set up by
1164 * by hfi1_alloc_devdata().
1165 */
1166void hfi1_free_devdata(struct hfi1_devdata *dd)
1167{
1168	struct hfi1_asic_data *ad;
1169	unsigned long flags;
1170
1171	xa_lock_irqsave(&hfi1_dev_table, flags);
1172	__xa_erase(&hfi1_dev_table, dd->unit);
1173	ad = release_asic_data(dd);
1174	xa_unlock_irqrestore(&hfi1_dev_table, flags);
1175
1176	finalize_asic_data(dd, ad);
1177	free_platform_config(dd);
1178	rcu_barrier(); /* wait for rcu callbacks to complete */
1179	free_percpu(dd->int_counter);
1180	free_percpu(dd->rcv_limit);
1181	free_percpu(dd->send_schedule);
1182	free_percpu(dd->tx_opstats);
1183	dd->int_counter   = NULL;
1184	dd->rcv_limit     = NULL;
1185	dd->send_schedule = NULL;
1186	dd->tx_opstats    = NULL;
1187	kfree(dd->comp_vect);
1188	dd->comp_vect = NULL;
1189	if (dd->rcvhdrtail_dummy_kvaddr)
1190		dma_free_coherent(&dd->pcidev->dev, sizeof(u64),
1191				  (void *)dd->rcvhdrtail_dummy_kvaddr,
1192				  dd->rcvhdrtail_dummy_dma);
1193	dd->rcvhdrtail_dummy_kvaddr = NULL;
1194	sdma_clean(dd, dd->num_sdma);
1195	rvt_dealloc_device(&dd->verbs_dev.rdi);
1196}
1197
1198/**
1199 * hfi1_alloc_devdata - Allocate our primary per-unit data structure.
1200 * @pdev: Valid PCI device
1201 * @extra: How many bytes to alloc past the default
1202 *
1203 * Must be done via verbs allocator, because the verbs cleanup process
1204 * both does cleanup and free of the data structure.
1205 * "extra" is for chip-specific data.
1206 */
1207static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,
1208					       size_t extra)
1209{
1210	struct hfi1_devdata *dd;
1211	int ret, nports;
1212
1213	/* extra is * number of ports */
1214	nports = extra / sizeof(struct hfi1_pportdata);
1215
1216	dd = (struct hfi1_devdata *)rvt_alloc_device(sizeof(*dd) + extra,
1217						     nports);
1218	if (!dd)
1219		return ERR_PTR(-ENOMEM);
1220	dd->num_pports = nports;
1221	dd->pport = (struct hfi1_pportdata *)(dd + 1);
1222	dd->pcidev = pdev;
1223	pci_set_drvdata(pdev, dd);
1224
1225	ret = xa_alloc_irq(&hfi1_dev_table, &dd->unit, dd, xa_limit_32b,
1226			GFP_KERNEL);
1227	if (ret < 0) {
1228		dev_err(&pdev->dev,
1229			"Could not allocate unit ID: error %d\n", -ret);
1230		goto bail;
1231	}
1232	rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s_%d", class_name(), dd->unit);
1233	/*
1234	 * If the BIOS does not have the NUMA node information set, select
1235	 * NUMA 0 so we get consistent performance.
1236	 */
1237	dd->node = pcibus_to_node(pdev->bus);
1238	if (dd->node == NUMA_NO_NODE) {
1239		dd_dev_err(dd, "Invalid PCI NUMA node. Performance may be affected\n");
1240		dd->node = 0;
1241	}
1242
1243	/*
1244	 * Initialize all locks for the device. This needs to be as early as
1245	 * possible so locks are usable.
1246	 */
1247	spin_lock_init(&dd->sc_lock);
1248	spin_lock_init(&dd->sendctrl_lock);
1249	spin_lock_init(&dd->rcvctrl_lock);
1250	spin_lock_init(&dd->uctxt_lock);
1251	spin_lock_init(&dd->hfi1_diag_trans_lock);
1252	spin_lock_init(&dd->sc_init_lock);
1253	spin_lock_init(&dd->dc8051_memlock);
1254	seqlock_init(&dd->sc2vl_lock);
1255	spin_lock_init(&dd->sde_map_lock);
1256	spin_lock_init(&dd->pio_map_lock);
1257	mutex_init(&dd->dc8051_lock);
1258	init_waitqueue_head(&dd->event_queue);
1259	spin_lock_init(&dd->irq_src_lock);
1260
1261	dd->int_counter = alloc_percpu(u64);
1262	if (!dd->int_counter) {
1263		ret = -ENOMEM;
1264		goto bail;
1265	}
1266
1267	dd->rcv_limit = alloc_percpu(u64);
1268	if (!dd->rcv_limit) {
1269		ret = -ENOMEM;
1270		goto bail;
1271	}
1272
1273	dd->send_schedule = alloc_percpu(u64);
1274	if (!dd->send_schedule) {
1275		ret = -ENOMEM;
1276		goto bail;
1277	}
1278
1279	dd->tx_opstats = alloc_percpu(struct hfi1_opcode_stats_perctx);
1280	if (!dd->tx_opstats) {
1281		ret = -ENOMEM;
1282		goto bail;
1283	}
1284
1285	dd->comp_vect = kzalloc(sizeof(*dd->comp_vect), GFP_KERNEL);
1286	if (!dd->comp_vect) {
1287		ret = -ENOMEM;
1288		goto bail;
1289	}
1290
1291	/* allocate dummy tail memory for all receive contexts */
1292	dd->rcvhdrtail_dummy_kvaddr =
1293		dma_alloc_coherent(&dd->pcidev->dev, sizeof(u64),
1294				   &dd->rcvhdrtail_dummy_dma, GFP_KERNEL);
1295	if (!dd->rcvhdrtail_dummy_kvaddr) {
1296		ret = -ENOMEM;
1297		goto bail;
1298	}
1299
1300	atomic_set(&dd->ipoib_rsm_usr_num, 0);
1301	return dd;
1302
1303bail:
1304	hfi1_free_devdata(dd);
1305	return ERR_PTR(ret);
1306}
1307
1308/*
1309 * Called from freeze mode handlers, and from PCI error
1310 * reporting code.  Should be paranoid about state of
1311 * system and data structures.
1312 */
1313void hfi1_disable_after_error(struct hfi1_devdata *dd)
1314{
1315	if (dd->flags & HFI1_INITTED) {
1316		u32 pidx;
1317
1318		dd->flags &= ~HFI1_INITTED;
1319		if (dd->pport)
1320			for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1321				struct hfi1_pportdata *ppd;
1322
1323				ppd = dd->pport + pidx;
1324				if (dd->flags & HFI1_PRESENT)
1325					set_link_state(ppd, HLS_DN_DISABLE);
1326
1327				if (ppd->statusp)
1328					*ppd->statusp &= ~HFI1_STATUS_IB_READY;
1329			}
1330	}
1331
1332	/*
1333	 * Mark as having had an error for driver, and also
1334	 * for /sys and status word mapped to user programs.
1335	 * This marks unit as not usable, until reset.
1336	 */
1337	if (dd->status)
1338		dd->status->dev |= HFI1_STATUS_HWERROR;
1339}
1340
1341static void remove_one(struct pci_dev *);
1342static int init_one(struct pci_dev *, const struct pci_device_id *);
1343static void shutdown_one(struct pci_dev *);
1344
1345#define DRIVER_LOAD_MSG "Cornelis " DRIVER_NAME " loaded: "
1346#define PFX DRIVER_NAME ": "
1347
1348const struct pci_device_id hfi1_pci_tbl[] = {
1349	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL0) },
1350	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL1) },
1351	{ 0, }
1352};
1353
1354MODULE_DEVICE_TABLE(pci, hfi1_pci_tbl);
1355
1356static struct pci_driver hfi1_pci_driver = {
1357	.name = DRIVER_NAME,
1358	.probe = init_one,
1359	.remove = remove_one,
1360	.shutdown = shutdown_one,
1361	.id_table = hfi1_pci_tbl,
1362	.err_handler = &hfi1_pci_err_handler,
1363};
1364
1365static void __init compute_krcvqs(void)
1366{
1367	int i;
1368
1369	for (i = 0; i < krcvqsset; i++)
1370		n_krcvqs += krcvqs[i];
1371}
1372
1373/*
1374 * Do all the generic driver unit- and chip-independent memory
1375 * allocation and initialization.
1376 */
1377static int __init hfi1_mod_init(void)
1378{
1379	int ret;
1380
1381	ret = dev_init();
1382	if (ret)
1383		goto bail;
1384
1385	ret = node_affinity_init();
1386	if (ret)
1387		goto bail;
1388
1389	/* validate max MTU before any devices start */
1390	if (!valid_opa_max_mtu(hfi1_max_mtu)) {
1391		pr_err("Invalid max_mtu 0x%x, using 0x%x instead\n",
1392		       hfi1_max_mtu, HFI1_DEFAULT_MAX_MTU);
1393		hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
1394	}
1395	/* valid CUs run from 1-128 in powers of 2 */
1396	if (hfi1_cu > 128 || !is_power_of_2(hfi1_cu))
1397		hfi1_cu = 1;
1398	/* valid credit return threshold is 0-100, variable is unsigned */
1399	if (user_credit_return_threshold > 100)
1400		user_credit_return_threshold = 100;
1401
1402	compute_krcvqs();
1403	/*
1404	 * sanitize receive interrupt count, time must wait until after
1405	 * the hardware type is known
1406	 */
1407	if (rcv_intr_count > RCV_HDR_HEAD_COUNTER_MASK)
1408		rcv_intr_count = RCV_HDR_HEAD_COUNTER_MASK;
1409	/* reject invalid combinations */
1410	if (rcv_intr_count == 0 && rcv_intr_timeout == 0) {
1411		pr_err("Invalid mode: both receive interrupt count and available timeout are zero - setting interrupt count to 1\n");
1412		rcv_intr_count = 1;
1413	}
1414	if (rcv_intr_count > 1 && rcv_intr_timeout == 0) {
1415		/*
1416		 * Avoid indefinite packet delivery by requiring a timeout
1417		 * if count is > 1.
1418		 */
1419		pr_err("Invalid mode: receive interrupt count greater than 1 and available timeout is zero - setting available timeout to 1\n");
1420		rcv_intr_timeout = 1;
1421	}
1422	if (rcv_intr_dynamic && !(rcv_intr_count > 1 && rcv_intr_timeout > 0)) {
1423		/*
1424		 * The dynamic algorithm expects a non-zero timeout
1425		 * and a count > 1.
1426		 */
1427		pr_err("Invalid mode: dynamic receive interrupt mitigation with invalid count and timeout - turning dynamic off\n");
1428		rcv_intr_dynamic = 0;
1429	}
1430
1431	/* sanitize link CRC options */
1432	link_crc_mask &= SUPPORTED_CRCS;
1433
1434	ret = opfn_init();
1435	if (ret < 0) {
1436		pr_err("Failed to allocate opfn_wq");
1437		goto bail_dev;
1438	}
1439
1440	/*
1441	 * These must be called before the driver is registered with
1442	 * the PCI subsystem.
1443	 */
1444	hfi1_dbg_init();
1445	ret = pci_register_driver(&hfi1_pci_driver);
1446	if (ret < 0) {
1447		pr_err("Unable to register driver: error %d\n", -ret);
1448		goto bail_dev;
1449	}
1450	goto bail; /* all OK */
1451
1452bail_dev:
1453	hfi1_dbg_exit();
1454	dev_cleanup();
1455bail:
1456	return ret;
1457}
1458
1459module_init(hfi1_mod_init);
1460
1461/*
1462 * Do the non-unit driver cleanup, memory free, etc. at unload.
1463 */
1464static void __exit hfi1_mod_cleanup(void)
1465{
1466	pci_unregister_driver(&hfi1_pci_driver);
1467	opfn_exit();
1468	node_affinity_destroy_all();
1469	hfi1_dbg_exit();
1470
1471	WARN_ON(!xa_empty(&hfi1_dev_table));
1472	dispose_firmware();	/* asymmetric with obtain_firmware() */
1473	dev_cleanup();
1474}
1475
1476module_exit(hfi1_mod_cleanup);
1477
1478/* this can only be called after a successful initialization */
1479static void cleanup_device_data(struct hfi1_devdata *dd)
1480{
1481	int ctxt;
1482	int pidx;
1483
1484	/* users can't do anything more with chip */
1485	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1486		struct hfi1_pportdata *ppd = &dd->pport[pidx];
1487		struct cc_state *cc_state;
1488		int i;
1489
1490		if (ppd->statusp)
1491			*ppd->statusp &= ~HFI1_STATUS_CHIP_PRESENT;
1492
1493		for (i = 0; i < OPA_MAX_SLS; i++)
1494			hrtimer_cancel(&ppd->cca_timer[i].hrtimer);
1495
1496		spin_lock(&ppd->cc_state_lock);
1497		cc_state = get_cc_state_protected(ppd);
1498		RCU_INIT_POINTER(ppd->cc_state, NULL);
1499		spin_unlock(&ppd->cc_state_lock);
1500
1501		if (cc_state)
1502			kfree_rcu(cc_state, rcu);
1503	}
1504
1505	free_credit_return(dd);
1506
1507	/*
1508	 * Free any resources still in use (usually just kernel contexts)
1509	 * at unload; we do for ctxtcnt, because that's what we allocate.
1510	 */
1511	for (ctxt = 0; dd->rcd && ctxt < dd->num_rcv_contexts; ctxt++) {
1512		struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
1513
1514		if (rcd) {
1515			hfi1_free_ctxt_rcv_groups(rcd);
1516			hfi1_free_ctxt(rcd);
1517		}
1518	}
1519
1520	kfree(dd->rcd);
1521	dd->rcd = NULL;
1522
1523	free_pio_map(dd);
1524	/* must follow rcv context free - need to remove rcv's hooks */
1525	for (ctxt = 0; ctxt < dd->num_send_contexts; ctxt++)
1526		sc_free(dd->send_contexts[ctxt].sc);
1527	dd->num_send_contexts = 0;
1528	kfree(dd->send_contexts);
1529	dd->send_contexts = NULL;
1530	kfree(dd->hw_to_sw);
1531	dd->hw_to_sw = NULL;
1532	kfree(dd->boardname);
1533	vfree(dd->events);
1534	vfree(dd->status);
1535}
1536
1537/*
1538 * Clean up on unit shutdown, or error during unit load after
1539 * successful initialization.
1540 */
1541static void postinit_cleanup(struct hfi1_devdata *dd)
1542{
1543	hfi1_start_cleanup(dd);
1544	hfi1_comp_vectors_clean_up(dd);
1545	hfi1_dev_affinity_clean_up(dd);
1546
1547	hfi1_pcie_ddcleanup(dd);
1548	hfi1_pcie_cleanup(dd->pcidev);
1549
1550	cleanup_device_data(dd);
1551
1552	hfi1_free_devdata(dd);
1553}
1554
1555static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1556{
1557	int ret = 0, j, pidx, initfail;
1558	struct hfi1_devdata *dd;
1559	struct hfi1_pportdata *ppd;
1560
1561	/* First, lock the non-writable module parameters */
1562	HFI1_CAP_LOCK();
1563
1564	/* Validate dev ids */
1565	if (!(ent->device == PCI_DEVICE_ID_INTEL0 ||
1566	      ent->device == PCI_DEVICE_ID_INTEL1)) {
1567		dev_err(&pdev->dev, "Failing on unknown Intel deviceid 0x%x\n",
1568			ent->device);
1569		ret = -ENODEV;
1570		goto bail;
1571	}
1572
1573	/* Allocate the dd so we can get to work */
1574	dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS *
1575				sizeof(struct hfi1_pportdata));
1576	if (IS_ERR(dd)) {
1577		ret = PTR_ERR(dd);
1578		goto bail;
1579	}
1580
1581	/* Validate some global module parameters */
1582	ret = hfi1_validate_rcvhdrcnt(dd, rcvhdrcnt);
1583	if (ret)
1584		goto bail;
1585
1586	/* use the encoding function as a sanitization check */
1587	if (!encode_rcv_header_entry_size(hfi1_hdrq_entsize)) {
1588		dd_dev_err(dd, "Invalid HdrQ Entry size %u\n",
1589			   hfi1_hdrq_entsize);
1590		ret = -EINVAL;
1591		goto bail;
1592	}
1593
1594	/* The receive eager buffer size must be set before the receive
1595	 * contexts are created.
1596	 *
1597	 * Set the eager buffer size.  Validate that it falls in a range
1598	 * allowed by the hardware - all powers of 2 between the min and
1599	 * max.  The maximum valid MTU is within the eager buffer range
1600	 * so we do not need to cap the max_mtu by an eager buffer size
1601	 * setting.
1602	 */
1603	if (eager_buffer_size) {
1604		if (!is_power_of_2(eager_buffer_size))
1605			eager_buffer_size =
1606				roundup_pow_of_two(eager_buffer_size);
1607		eager_buffer_size =
1608			clamp_val(eager_buffer_size,
1609				  MIN_EAGER_BUFFER * 8,
1610				  MAX_EAGER_BUFFER_TOTAL);
1611		dd_dev_info(dd, "Eager buffer size %u\n",
1612			    eager_buffer_size);
1613	} else {
1614		dd_dev_err(dd, "Invalid Eager buffer size of 0\n");
1615		ret = -EINVAL;
1616		goto bail;
1617	}
1618
1619	/* restrict value of hfi1_rcvarr_split */
1620	hfi1_rcvarr_split = clamp_val(hfi1_rcvarr_split, 0, 100);
1621
1622	ret = hfi1_pcie_init(dd);
1623	if (ret)
1624		goto bail;
1625
1626	/*
1627	 * Do device-specific initialization, function table setup, dd
1628	 * allocation, etc.
1629	 */
1630	ret = hfi1_init_dd(dd);
1631	if (ret)
1632		goto clean_bail; /* error already printed */
1633
1634	ret = create_workqueues(dd);
1635	if (ret)
1636		goto clean_bail;
1637
1638	/* do the generic initialization */
1639	initfail = hfi1_init(dd, 0);
1640
1641	ret = hfi1_register_ib_device(dd);
1642
1643	/*
1644	 * Now ready for use.  this should be cleared whenever we
1645	 * detect a reset, or initiate one.  If earlier failure,
1646	 * we still create devices, so diags, etc. can be used
1647	 * to determine cause of problem.
1648	 */
1649	if (!initfail && !ret) {
1650		dd->flags |= HFI1_INITTED;
1651		/* create debufs files after init and ib register */
1652		hfi1_dbg_ibdev_init(&dd->verbs_dev);
1653	}
1654
1655	j = hfi1_device_create(dd);
1656	if (j)
1657		dd_dev_err(dd, "Failed to create /dev devices: %d\n", -j);
1658
1659	if (initfail || ret) {
1660		msix_clean_up_interrupts(dd);
1661		stop_timers(dd);
1662		flush_workqueue(ib_wq);
1663		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1664			hfi1_quiet_serdes(dd->pport + pidx);
1665			ppd = dd->pport + pidx;
1666			if (ppd->hfi1_wq) {
1667				destroy_workqueue(ppd->hfi1_wq);
1668				ppd->hfi1_wq = NULL;
1669			}
1670			if (ppd->link_wq) {
1671				destroy_workqueue(ppd->link_wq);
1672				ppd->link_wq = NULL;
1673			}
1674		}
1675		if (!j)
1676			hfi1_device_remove(dd);
1677		if (!ret)
1678			hfi1_unregister_ib_device(dd);
1679		postinit_cleanup(dd);
1680		if (initfail)
1681			ret = initfail;
1682		goto bail;	/* everything already cleaned */
1683	}
1684
1685	sdma_start(dd);
1686
1687	return 0;
1688
1689clean_bail:
1690	hfi1_pcie_cleanup(pdev);
1691bail:
1692	return ret;
1693}
1694
1695static void wait_for_clients(struct hfi1_devdata *dd)
1696{
1697	/*
1698	 * Remove the device init value and complete the device if there is
1699	 * no clients or wait for active clients to finish.
1700	 */
1701	if (refcount_dec_and_test(&dd->user_refcount))
1702		complete(&dd->user_comp);
1703
1704	wait_for_completion(&dd->user_comp);
1705}
1706
1707static void remove_one(struct pci_dev *pdev)
1708{
1709	struct hfi1_devdata *dd = pci_get_drvdata(pdev);
1710
1711	/* close debugfs files before ib unregister */
1712	hfi1_dbg_ibdev_exit(&dd->verbs_dev);
1713
1714	/* remove the /dev hfi1 interface */
1715	hfi1_device_remove(dd);
1716
1717	/* wait for existing user space clients to finish */
1718	wait_for_clients(dd);
1719
1720	/* unregister from IB core */
1721	hfi1_unregister_ib_device(dd);
1722
1723	/* free netdev data */
1724	hfi1_free_rx(dd);
1725
1726	/*
1727	 * Disable the IB link, disable interrupts on the device,
1728	 * clear dma engines, etc.
1729	 */
1730	shutdown_device(dd);
1731	destroy_workqueues(dd);
1732
1733	stop_timers(dd);
1734
1735	/* wait until all of our (qsfp) queue_work() calls complete */
1736	flush_workqueue(ib_wq);
1737
1738	postinit_cleanup(dd);
1739}
1740
1741static void shutdown_one(struct pci_dev *pdev)
1742{
1743	struct hfi1_devdata *dd = pci_get_drvdata(pdev);
1744
1745	shutdown_device(dd);
1746}
1747
1748/**
1749 * hfi1_create_rcvhdrq - create a receive header queue
1750 * @dd: the hfi1_ib device
1751 * @rcd: the context data
1752 *
1753 * This must be contiguous memory (from an i/o perspective), and must be
1754 * DMA'able (which means for some systems, it will go through an IOMMU,
1755 * or be forced into a low address range).
1756 */
1757int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
1758{
1759	unsigned amt;
1760
1761	if (!rcd->rcvhdrq) {
1762		amt = rcvhdrq_size(rcd);
1763
1764		rcd->rcvhdrq = dma_alloc_coherent(&dd->pcidev->dev, amt,
1765						  &rcd->rcvhdrq_dma,
1766						  GFP_KERNEL);
1767
1768		if (!rcd->rcvhdrq) {
1769			dd_dev_err(dd,
1770				   "attempt to allocate %d bytes for ctxt %u rcvhdrq failed\n",
1771				   amt, rcd->ctxt);
1772			goto bail;
1773		}
1774
1775		if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ||
1776		    HFI1_CAP_UGET_MASK(rcd->flags, DMA_RTAIL)) {
1777			rcd->rcvhdrtail_kvaddr = dma_alloc_coherent(&dd->pcidev->dev,
1778								    PAGE_SIZE,
1779								    &rcd->rcvhdrqtailaddr_dma,
1780								    GFP_KERNEL);
1781			if (!rcd->rcvhdrtail_kvaddr)
1782				goto bail_free;
1783		}
1784	}
1785
1786	set_hdrq_regs(rcd->dd, rcd->ctxt, rcd->rcvhdrqentsize,
1787		      rcd->rcvhdrq_cnt);
1788
1789	return 0;
1790
1791bail_free:
1792	dd_dev_err(dd,
1793		   "attempt to allocate 1 page for ctxt %u rcvhdrqtailaddr failed\n",
1794		   rcd->ctxt);
1795	dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq,
1796			  rcd->rcvhdrq_dma);
1797	rcd->rcvhdrq = NULL;
1798bail:
1799	return -ENOMEM;
1800}
1801
1802/**
1803 * hfi1_setup_eagerbufs - llocate eager buffers, both kernel and user
1804 * contexts.
1805 * @rcd: the context we are setting up.
1806 *
1807 * Allocate the eager TID buffers and program them into hip.
1808 * They are no longer completely contiguous, we do multiple allocation
1809 * calls.  Otherwise we get the OOM code involved, by asking for too
1810 * much per call, with disastrous results on some kernels.
1811 */
1812int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)
1813{
1814	struct hfi1_devdata *dd = rcd->dd;
1815	u32 max_entries, egrtop, alloced_bytes = 0;
1816	u16 order, idx = 0;
1817	int ret = 0;
1818	u16 round_mtu = roundup_pow_of_two(hfi1_max_mtu);
1819
1820	/*
1821	 * The minimum size of the eager buffers is a groups of MTU-sized
1822	 * buffers.
1823	 * The global eager_buffer_size parameter is checked against the
1824	 * theoretical lower limit of the value. Here, we check against the
1825	 * MTU.
1826	 */
1827	if (rcd->egrbufs.size < (round_mtu * dd->rcv_entries.group_size))
1828		rcd->egrbufs.size = round_mtu * dd->rcv_entries.group_size;
1829	/*
1830	 * If using one-pkt-per-egr-buffer, lower the eager buffer
1831	 * size to the max MTU (page-aligned).
1832	 */
1833	if (!HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR))
1834		rcd->egrbufs.rcvtid_size = round_mtu;
1835
1836	/*
1837	 * Eager buffers sizes of 1MB or less require smaller TID sizes
1838	 * to satisfy the "multiple of 8 RcvArray entries" requirement.
1839	 */
1840	if (rcd->egrbufs.size <= (1 << 20))
1841		rcd->egrbufs.rcvtid_size = max((unsigned long)round_mtu,
1842			rounddown_pow_of_two(rcd->egrbufs.size / 8));
1843
1844	while (alloced_bytes < rcd->egrbufs.size &&
1845	       rcd->egrbufs.alloced < rcd->egrbufs.count) {
1846		rcd->egrbufs.buffers[idx].addr =
1847			dma_alloc_coherent(&dd->pcidev->dev,
1848					   rcd->egrbufs.rcvtid_size,
1849					   &rcd->egrbufs.buffers[idx].dma,
1850					   GFP_KERNEL);
1851		if (rcd->egrbufs.buffers[idx].addr) {
1852			rcd->egrbufs.buffers[idx].len =
1853				rcd->egrbufs.rcvtid_size;
1854			rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].addr =
1855				rcd->egrbufs.buffers[idx].addr;
1856			rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].dma =
1857				rcd->egrbufs.buffers[idx].dma;
1858			rcd->egrbufs.alloced++;
1859			alloced_bytes += rcd->egrbufs.rcvtid_size;
1860			idx++;
1861		} else {
1862			u32 new_size, i, j;
1863			u64 offset = 0;
1864
1865			/*
1866			 * Fail the eager buffer allocation if:
1867			 *   - we are already using the lowest acceptable size
1868			 *   - we are using one-pkt-per-egr-buffer (this implies
1869			 *     that we are accepting only one size)
1870			 */
1871			if (rcd->egrbufs.rcvtid_size == round_mtu ||
1872			    !HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR)) {
1873				dd_dev_err(dd, "ctxt%u: Failed to allocate eager buffers\n",
1874					   rcd->ctxt);
1875				ret = -ENOMEM;
1876				goto bail_rcvegrbuf_phys;
1877			}
1878
1879			new_size = rcd->egrbufs.rcvtid_size / 2;
1880
1881			/*
1882			 * If the first attempt to allocate memory failed, don't
1883			 * fail everything but continue with the next lower
1884			 * size.
1885			 */
1886			if (idx == 0) {
1887				rcd->egrbufs.rcvtid_size = new_size;
1888				continue;
1889			}
1890
1891			/*
1892			 * Re-partition already allocated buffers to a smaller
1893			 * size.
1894			 */
1895			rcd->egrbufs.alloced = 0;
1896			for (i = 0, j = 0, offset = 0; j < idx; i++) {
1897				if (i >= rcd->egrbufs.count)
1898					break;
1899				rcd->egrbufs.rcvtids[i].dma =
1900					rcd->egrbufs.buffers[j].dma + offset;
1901				rcd->egrbufs.rcvtids[i].addr =
1902					rcd->egrbufs.buffers[j].addr + offset;
1903				rcd->egrbufs.alloced++;
1904				if ((rcd->egrbufs.buffers[j].dma + offset +
1905				     new_size) ==
1906				    (rcd->egrbufs.buffers[j].dma +
1907				     rcd->egrbufs.buffers[j].len)) {
1908					j++;
1909					offset = 0;
1910				} else {
1911					offset += new_size;
1912				}
1913			}
1914			rcd->egrbufs.rcvtid_size = new_size;
1915		}
1916	}
1917	rcd->egrbufs.numbufs = idx;
1918	rcd->egrbufs.size = alloced_bytes;
1919
1920	hfi1_cdbg(PROC,
1921		  "ctxt%u: Alloced %u rcv tid entries @ %uKB, total %uKB",
1922		  rcd->ctxt, rcd->egrbufs.alloced,
1923		  rcd->egrbufs.rcvtid_size / 1024, rcd->egrbufs.size / 1024);
1924
1925	/*
1926	 * Set the contexts rcv array head update threshold to the closest
1927	 * power of 2 (so we can use a mask instead of modulo) below half
1928	 * the allocated entries.
1929	 */
1930	rcd->egrbufs.threshold =
1931		rounddown_pow_of_two(rcd->egrbufs.alloced / 2);
1932	/*
1933	 * Compute the expected RcvArray entry base. This is done after
1934	 * allocating the eager buffers in order to maximize the
1935	 * expected RcvArray entries for the context.
1936	 */
1937	max_entries = rcd->rcv_array_groups * dd->rcv_entries.group_size;
1938	egrtop = roundup(rcd->egrbufs.alloced, dd->rcv_entries.group_size);
1939	rcd->expected_count = max_entries - egrtop;
1940	if (rcd->expected_count > MAX_TID_PAIR_ENTRIES * 2)
1941		rcd->expected_count = MAX_TID_PAIR_ENTRIES * 2;
1942
1943	rcd->expected_base = rcd->eager_base + egrtop;
1944	hfi1_cdbg(PROC, "ctxt%u: eager:%u, exp:%u, egrbase:%u, expbase:%u",
1945		  rcd->ctxt, rcd->egrbufs.alloced, rcd->expected_count,
1946		  rcd->eager_base, rcd->expected_base);
1947
1948	if (!hfi1_rcvbuf_validate(rcd->egrbufs.rcvtid_size, PT_EAGER, &order)) {
1949		hfi1_cdbg(PROC,
1950			  "ctxt%u: current Eager buffer size is invalid %u",
1951			  rcd->ctxt, rcd->egrbufs.rcvtid_size);
1952		ret = -EINVAL;
1953		goto bail_rcvegrbuf_phys;
1954	}
1955
1956	for (idx = 0; idx < rcd->egrbufs.alloced; idx++) {
1957		hfi1_put_tid(dd, rcd->eager_base + idx, PT_EAGER,
1958			     rcd->egrbufs.rcvtids[idx].dma, order);
1959		cond_resched();
1960	}
1961
1962	return 0;
1963
1964bail_rcvegrbuf_phys:
1965	for (idx = 0; idx < rcd->egrbufs.alloced &&
1966	     rcd->egrbufs.buffers[idx].addr;
1967	     idx++) {
1968		dma_free_coherent(&dd->pcidev->dev,
1969				  rcd->egrbufs.buffers[idx].len,
1970				  rcd->egrbufs.buffers[idx].addr,
1971				  rcd->egrbufs.buffers[idx].dma);
1972		rcd->egrbufs.buffers[idx].addr = NULL;
1973		rcd->egrbufs.buffers[idx].dma = 0;
1974		rcd->egrbufs.buffers[idx].len = 0;
1975	}
1976
1977	return ret;
1978}