sdma.c

   1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2/*
   3 * Copyright(c) 2015 - 2018 Intel Corporation.
   4 */
   5
   6#include <linux/spinlock.h>
   7#include <linux/seqlock.h>
   8#include <linux/netdevice.h>
   9#include <linux/moduleparam.h>
  10#include <linux/bitops.h>
  11#include <linux/timer.h>
  12#include <linux/vmalloc.h>
  13#include <linux/highmem.h>
  14
  15#include "hfi.h"
  16#include "common.h"
  17#include "qp.h"
  18#include "sdma.h"
  19#include "iowait.h"
  20#include "trace.h"
  21
  22/* must be a power of 2 >= 64 <= 32768 */
  23#define SDMA_DESCQ_CNT 2048
  24#define SDMA_DESC_INTR 64
  25#define INVALID_TAIL 0xffff
  26#define SDMA_PAD max_t(size_t, MAX_16B_PADDING, sizeof(u32))
  27
  28static uint sdma_descq_cnt = SDMA_DESCQ_CNT;
  29module_param(sdma_descq_cnt, uint, S_IRUGO);
  30MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries");
  31
  32static uint sdma_idle_cnt = 250;
  33module_param(sdma_idle_cnt, uint, S_IRUGO);
  34MODULE_PARM_DESC(sdma_idle_cnt, "sdma interrupt idle delay (ns,default 250)");
  35
  36uint mod_num_sdma;
  37module_param_named(num_sdma, mod_num_sdma, uint, S_IRUGO);
  38MODULE_PARM_DESC(num_sdma, "Set max number SDMA engines to use");
  39
  40static uint sdma_desct_intr = SDMA_DESC_INTR;
  41module_param_named(desct_intr, sdma_desct_intr, uint, S_IRUGO | S_IWUSR);
  42MODULE_PARM_DESC(desct_intr, "Number of SDMA descriptor before interrupt");
  43
  44#define SDMA_WAIT_BATCH_SIZE 20
  45/* max wait time for a SDMA engine to indicate it has halted */
  46#define SDMA_ERR_HALT_TIMEOUT 10 /* ms */
  47/* all SDMA engine errors that cause a halt */
  48
  49#define SD(name) SEND_DMA_##name
  50#define ALL_SDMA_ENG_HALT_ERRS \
  51	(SD(ENG_ERR_STATUS_SDMA_WRONG_DW_ERR_SMASK) \
  52	| SD(ENG_ERR_STATUS_SDMA_GEN_MISMATCH_ERR_SMASK) \
  53	| SD(ENG_ERR_STATUS_SDMA_TOO_LONG_ERR_SMASK) \
  54	| SD(ENG_ERR_STATUS_SDMA_TAIL_OUT_OF_BOUNDS_ERR_SMASK) \
  55	| SD(ENG_ERR_STATUS_SDMA_FIRST_DESC_ERR_SMASK) \
  56	| SD(ENG_ERR_STATUS_SDMA_MEM_READ_ERR_SMASK) \
  57	| SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK) \
  58	| SD(ENG_ERR_STATUS_SDMA_LENGTH_MISMATCH_ERR_SMASK) \
  59	| SD(ENG_ERR_STATUS_SDMA_PACKET_DESC_OVERFLOW_ERR_SMASK) \
  60	| SD(ENG_ERR_STATUS_SDMA_HEADER_SELECT_ERR_SMASK) \
  61	| SD(ENG_ERR_STATUS_SDMA_HEADER_ADDRESS_ERR_SMASK) \
  62	| SD(ENG_ERR_STATUS_SDMA_HEADER_LENGTH_ERR_SMASK) \
  63	| SD(ENG_ERR_STATUS_SDMA_TIMEOUT_ERR_SMASK) \
  64	| SD(ENG_ERR_STATUS_SDMA_DESC_TABLE_UNC_ERR_SMASK) \
  65	| SD(ENG_ERR_STATUS_SDMA_ASSEMBLY_UNC_ERR_SMASK) \
  66	| SD(ENG_ERR_STATUS_SDMA_PACKET_TRACKING_UNC_ERR_SMASK) \
  67	| SD(ENG_ERR_STATUS_SDMA_HEADER_STORAGE_UNC_ERR_SMASK) \
  68	| SD(ENG_ERR_STATUS_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SMASK))
  69
  70/* sdma_sendctrl operations */
  71#define SDMA_SENDCTRL_OP_ENABLE    BIT(0)
  72#define SDMA_SENDCTRL_OP_INTENABLE BIT(1)
  73#define SDMA_SENDCTRL_OP_HALT      BIT(2)
  74#define SDMA_SENDCTRL_OP_CLEANUP   BIT(3)
  75
  76/* handle long defines */
  77#define SDMA_EGRESS_PACKET_OCCUPANCY_SMASK \
  78SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SMASK
  79#define SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT \
  80SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT
  81
  82static const char * const sdma_state_names[] = {
  83	[sdma_state_s00_hw_down]                = "s00_HwDown",
  84	[sdma_state_s10_hw_start_up_halt_wait]  = "s10_HwStartUpHaltWait",
  85	[sdma_state_s15_hw_start_up_clean_wait] = "s15_HwStartUpCleanWait",
  86	[sdma_state_s20_idle]                   = "s20_Idle",
  87	[sdma_state_s30_sw_clean_up_wait]       = "s30_SwCleanUpWait",
  88	[sdma_state_s40_hw_clean_up_wait]       = "s40_HwCleanUpWait",
  89	[sdma_state_s50_hw_halt_wait]           = "s50_HwHaltWait",
  90	[sdma_state_s60_idle_halt_wait]         = "s60_IdleHaltWait",
  91	[sdma_state_s80_hw_freeze]		= "s80_HwFreeze",
  92	[sdma_state_s82_freeze_sw_clean]	= "s82_FreezeSwClean",
  93	[sdma_state_s99_running]                = "s99_Running",
  94};
  95
  96#ifdef CONFIG_SDMA_VERBOSITY
  97static const char * const sdma_event_names[] = {
  98	[sdma_event_e00_go_hw_down]   = "e00_GoHwDown",
  99	[sdma_event_e10_go_hw_start]  = "e10_GoHwStart",
 100	[sdma_event_e15_hw_halt_done] = "e15_HwHaltDone",
 101	[sdma_event_e25_hw_clean_up_done] = "e25_HwCleanUpDone",
 102	[sdma_event_e30_go_running]   = "e30_GoRunning",
 103	[sdma_event_e40_sw_cleaned]   = "e40_SwCleaned",
 104	[sdma_event_e50_hw_cleaned]   = "e50_HwCleaned",
 105	[sdma_event_e60_hw_halted]    = "e60_HwHalted",
 106	[sdma_event_e70_go_idle]      = "e70_GoIdle",
 107	[sdma_event_e80_hw_freeze]    = "e80_HwFreeze",
 108	[sdma_event_e81_hw_frozen]    = "e81_HwFrozen",
 109	[sdma_event_e82_hw_unfreeze]  = "e82_HwUnfreeze",
 110	[sdma_event_e85_link_down]    = "e85_LinkDown",
 111	[sdma_event_e90_sw_halted]    = "e90_SwHalted",
 112};
 113#endif
 114
 115static const struct sdma_set_state_action sdma_action_table[] = {
 116	[sdma_state_s00_hw_down] = {
 117		.go_s99_running_tofalse = 1,
 118		.op_enable = 0,
 119		.op_intenable = 0,
 120		.op_halt = 0,
 121		.op_cleanup = 0,
 122	},
 123	[sdma_state_s10_hw_start_up_halt_wait] = {
 124		.op_enable = 0,
 125		.op_intenable = 0,
 126		.op_halt = 1,
 127		.op_cleanup = 0,
 128	},
 129	[sdma_state_s15_hw_start_up_clean_wait] = {
 130		.op_enable = 0,
 131		.op_intenable = 1,
 132		.op_halt = 0,
 133		.op_cleanup = 1,
 134	},
 135	[sdma_state_s20_idle] = {
 136		.op_enable = 0,
 137		.op_intenable = 1,
 138		.op_halt = 0,
 139		.op_cleanup = 0,
 140	},
 141	[sdma_state_s30_sw_clean_up_wait] = {
 142		.op_enable = 0,
 143		.op_intenable = 0,
 144		.op_halt = 0,
 145		.op_cleanup = 0,
 146	},
 147	[sdma_state_s40_hw_clean_up_wait] = {
 148		.op_enable = 0,
 149		.op_intenable = 0,
 150		.op_halt = 0,
 151		.op_cleanup = 1,
 152	},
 153	[sdma_state_s50_hw_halt_wait] = {
 154		.op_enable = 0,
 155		.op_intenable = 0,
 156		.op_halt = 0,
 157		.op_cleanup = 0,
 158	},
 159	[sdma_state_s60_idle_halt_wait] = {
 160		.go_s99_running_tofalse = 1,
 161		.op_enable = 0,
 162		.op_intenable = 0,
 163		.op_halt = 1,
 164		.op_cleanup = 0,
 165	},
 166	[sdma_state_s80_hw_freeze] = {
 167		.op_enable = 0,
 168		.op_intenable = 0,
 169		.op_halt = 0,
 170		.op_cleanup = 0,
 171	},
 172	[sdma_state_s82_freeze_sw_clean] = {
 173		.op_enable = 0,
 174		.op_intenable = 0,
 175		.op_halt = 0,
 176		.op_cleanup = 0,
 177	},
 178	[sdma_state_s99_running] = {
 179		.op_enable = 1,
 180		.op_intenable = 1,
 181		.op_halt = 0,
 182		.op_cleanup = 0,
 183		.go_s99_running_totrue = 1,
 184	},
 185};
 186
 187#define SDMA_TAIL_UPDATE_THRESH 0x1F
 188
 189/* declare all statics here rather than keep sorting */
 190static void sdma_complete(struct kref *);
 191static void sdma_finalput(struct sdma_state *);
 192static void sdma_get(struct sdma_state *);
 193static void sdma_hw_clean_up_task(struct tasklet_struct *);
 194static void sdma_put(struct sdma_state *);
 195static void sdma_set_state(struct sdma_engine *, enum sdma_states);
 196static void sdma_start_hw_clean_up(struct sdma_engine *);
 197static void sdma_sw_clean_up_task(struct tasklet_struct *);
 198static void sdma_sendctrl(struct sdma_engine *, unsigned);
 199static void init_sdma_regs(struct sdma_engine *, u32, uint);
 200static void sdma_process_event(
 201	struct sdma_engine *sde,
 202	enum sdma_events event);
 203static void __sdma_process_event(
 204	struct sdma_engine *sde,
 205	enum sdma_events event);
 206static void dump_sdma_state(struct sdma_engine *sde);
 207static void sdma_make_progress(struct sdma_engine *sde, u64 status);
 208static void sdma_desc_avail(struct sdma_engine *sde, uint avail);
 209static void sdma_flush_descq(struct sdma_engine *sde);
 210
 211/**
 212 * sdma_state_name() - return state string from enum
 213 * @state: state
 214 */
 215static const char *sdma_state_name(enum sdma_states state)
 216{
 217	return sdma_state_names[state];
 218}
 219
 220static void sdma_get(struct sdma_state *ss)
 221{
 222	kref_get(&ss->kref);
 223}
 224
 225static void sdma_complete(struct kref *kref)
 226{
 227	struct sdma_state *ss =
 228		container_of(kref, struct sdma_state, kref);
 229
 230	complete(&ss->comp);
 231}
 232
 233static void sdma_put(struct sdma_state *ss)
 234{
 235	kref_put(&ss->kref, sdma_complete);
 236}
 237
 238static void sdma_finalput(struct sdma_state *ss)
 239{
 240	sdma_put(ss);
 241	wait_for_completion(&ss->comp);
 242}
 243
 244static inline void write_sde_csr(
 245	struct sdma_engine *sde,
 246	u32 offset0,
 247	u64 value)
 248{
 249	write_kctxt_csr(sde->dd, sde->this_idx, offset0, value);
 250}
 251
 252static inline u64 read_sde_csr(
 253	struct sdma_engine *sde,
 254	u32 offset0)
 255{
 256	return read_kctxt_csr(sde->dd, sde->this_idx, offset0);
 257}
 258
 259/*
 260 * sdma_wait_for_packet_egress() - wait for the VL FIFO occupancy for
 261 * sdma engine 'sde' to drop to 0.
 262 */
 263static void sdma_wait_for_packet_egress(struct sdma_engine *sde,
 264					int pause)
 265{
 266	u64 off = 8 * sde->this_idx;
 267	struct hfi1_devdata *dd = sde->dd;
 268	int lcnt = 0;
 269	u64 reg_prev;
 270	u64 reg = 0;
 271
 272	while (1) {
 273		reg_prev = reg;
 274		reg = read_csr(dd, off + SEND_EGRESS_SEND_DMA_STATUS);
 275
 276		reg &= SDMA_EGRESS_PACKET_OCCUPANCY_SMASK;
 277		reg >>= SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT;
 278		if (reg == 0)
 279			break;
 280		/* counter is reest if accupancy count changes */
 281		if (reg != reg_prev)
 282			lcnt = 0;
 283		if (lcnt++ > 500) {
 284			/* timed out - bounce the link */
 285			dd_dev_err(dd, "%s: engine %u timeout waiting for packets to egress, remaining count %u, bouncing link\n",
 286				   __func__, sde->this_idx, (u32)reg);
 287			queue_work(dd->pport->link_wq,
 288				   &dd->pport->link_bounce_work);
 289			break;
 290		}
 291		udelay(1);
 292	}
 293}
 294
 295/*
 296 * sdma_wait() - wait for packet egress to complete for all SDMA engines,
 297 * and pause for credit return.
 298 */
 299void sdma_wait(struct hfi1_devdata *dd)
 300{
 301	int i;
 302
 303	for (i = 0; i < dd->num_sdma; i++) {
 304		struct sdma_engine *sde = &dd->per_sdma[i];
 305
 306		sdma_wait_for_packet_egress(sde, 0);
 307	}
 308}
 309
 310static inline void sdma_set_desc_cnt(struct sdma_engine *sde, unsigned cnt)
 311{
 312	u64 reg;
 313
 314	if (!(sde->dd->flags & HFI1_HAS_SDMA_TIMEOUT))
 315		return;
 316	reg = cnt;
 317	reg &= SD(DESC_CNT_CNT_MASK);
 318	reg <<= SD(DESC_CNT_CNT_SHIFT);
 319	write_sde_csr(sde, SD(DESC_CNT), reg);
 320}
 321
 322static inline void complete_tx(struct sdma_engine *sde,
 323			       struct sdma_txreq *tx,
 324			       int res)
 325{
 326	/* protect against complete modifying */
 327	struct iowait *wait = tx->wait;
 328	callback_t complete = tx->complete;
 329
 330#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
 331	trace_hfi1_sdma_out_sn(sde, tx->sn);
 332	if (WARN_ON_ONCE(sde->head_sn != tx->sn))
 333		dd_dev_err(sde->dd, "expected %llu got %llu\n",
 334			   sde->head_sn, tx->sn);
 335	sde->head_sn++;
 336#endif
 337	__sdma_txclean(sde->dd, tx);
 338	if (complete)
 339		(*complete)(tx, res);
 340	if (iowait_sdma_dec(wait))
 341		iowait_drain_wakeup(wait);
 342}
 343
 344/*
 345 * Complete all the sdma requests with a SDMA_TXREQ_S_ABORTED status
 346 *
 347 * Depending on timing there can be txreqs in two places:
 348 * - in the descq ring
 349 * - in the flush list
 350 *
 351 * To avoid ordering issues the descq ring needs to be flushed
 352 * first followed by the flush list.
 353 *
 354 * This routine is called from two places
 355 * - From a work queue item
 356 * - Directly from the state machine just before setting the
 357 *   state to running
 358 *
 359 * Must be called with head_lock held
 360 *
 361 */
 362static void sdma_flush(struct sdma_engine *sde)
 363{
 364	struct sdma_txreq *txp, *txp_next;
 365	LIST_HEAD(flushlist);
 366	unsigned long flags;
 367	uint seq;
 368
 369	/* flush from head to tail */
 370	sdma_flush_descq(sde);
 371	spin_lock_irqsave(&sde->flushlist_lock, flags);
 372	/* copy flush list */
 373	list_splice_init(&sde->flushlist, &flushlist);
 374	spin_unlock_irqrestore(&sde->flushlist_lock, flags);
 375	/* flush from flush list */
 376	list_for_each_entry_safe(txp, txp_next, &flushlist, list)
 377		complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED);
 378	/* wakeup QPs orphaned on the dmawait list */
 379	do {
 380		struct iowait *w, *nw;
 381
 382		seq = read_seqbegin(&sde->waitlock);
 383		if (!list_empty(&sde->dmawait)) {
 384			write_seqlock(&sde->waitlock);
 385			list_for_each_entry_safe(w, nw, &sde->dmawait, list) {
 386				if (w->wakeup) {
 387					w->wakeup(w, SDMA_AVAIL_REASON);
 388					list_del_init(&w->list);
 389				}
 390			}
 391			write_sequnlock(&sde->waitlock);
 392		}
 393	} while (read_seqretry(&sde->waitlock, seq));
 394}
 395
 396/*
 397 * Fields a work request for flushing the descq ring
 398 * and the flush list
 399 *
 400 * If the engine has been brought to running during
 401 * the scheduling delay, the flush is ignored, assuming
 402 * that the process of bringing the engine to running
 403 * would have done this flush prior to going to running.
 404 *
 405 */
 406static void sdma_field_flush(struct work_struct *work)
 407{
 408	unsigned long flags;
 409	struct sdma_engine *sde =
 410		container_of(work, struct sdma_engine, flush_worker);
 411
 412	write_seqlock_irqsave(&sde->head_lock, flags);
 413	if (!__sdma_running(sde))
 414		sdma_flush(sde);
 415	write_sequnlock_irqrestore(&sde->head_lock, flags);
 416}
 417
 418static void sdma_err_halt_wait(struct work_struct *work)
 419{
 420	struct sdma_engine *sde = container_of(work, struct sdma_engine,
 421						err_halt_worker);
 422	u64 statuscsr;
 423	unsigned long timeout;
 424
 425	timeout = jiffies + msecs_to_jiffies(SDMA_ERR_HALT_TIMEOUT);
 426	while (1) {
 427		statuscsr = read_sde_csr(sde, SD(STATUS));
 428		statuscsr &= SD(STATUS_ENG_HALTED_SMASK);
 429		if (statuscsr)
 430			break;
 431		if (time_after(jiffies, timeout)) {
 432			dd_dev_err(sde->dd,
 433				   "SDMA engine %d - timeout waiting for engine to halt\n",
 434				   sde->this_idx);
 435			/*
 436			 * Continue anyway.  This could happen if there was
 437			 * an uncorrectable error in the wrong spot.
 438			 */
 439			break;
 440		}
 441		usleep_range(80, 120);
 442	}
 443
 444	sdma_process_event(sde, sdma_event_e15_hw_halt_done);
 445}
 446
 447static void sdma_err_progress_check_schedule(struct sdma_engine *sde)
 448{
 449	if (!is_bx(sde->dd) && HFI1_CAP_IS_KSET(SDMA_AHG)) {
 450		unsigned index;
 451		struct hfi1_devdata *dd = sde->dd;
 452
 453		for (index = 0; index < dd->num_sdma; index++) {
 454			struct sdma_engine *curr_sdma = &dd->per_sdma[index];
 455
 456			if (curr_sdma != sde)
 457				curr_sdma->progress_check_head =
 458							curr_sdma->descq_head;
 459		}
 460		dd_dev_err(sde->dd,
 461			   "SDMA engine %d - check scheduled\n",
 462				sde->this_idx);
 463		mod_timer(&sde->err_progress_check_timer, jiffies + 10);
 464	}
 465}
 466
 467static void sdma_err_progress_check(struct timer_list *t)
 468{
 469	unsigned index;
 470	struct sdma_engine *sde = timer_container_of(sde, t,
 471						     err_progress_check_timer);
 472
 473	dd_dev_err(sde->dd, "SDE progress check event\n");
 474	for (index = 0; index < sde->dd->num_sdma; index++) {
 475		struct sdma_engine *curr_sde = &sde->dd->per_sdma[index];
 476		unsigned long flags;
 477
 478		/* check progress on each engine except the current one */
 479		if (curr_sde == sde)
 480			continue;
 481		/*
 482		 * We must lock interrupts when acquiring sde->lock,
 483		 * to avoid a deadlock if interrupt triggers and spins on
 484		 * the same lock on same CPU
 485		 */
 486		spin_lock_irqsave(&curr_sde->tail_lock, flags);
 487		write_seqlock(&curr_sde->head_lock);
 488
 489		/* skip non-running queues */
 490		if (curr_sde->state.current_state != sdma_state_s99_running) {
 491			write_sequnlock(&curr_sde->head_lock);
 492			spin_unlock_irqrestore(&curr_sde->tail_lock, flags);
 493			continue;
 494		}
 495
 496		if ((curr_sde->descq_head != curr_sde->descq_tail) &&
 497		    (curr_sde->descq_head ==
 498				curr_sde->progress_check_head))
 499			__sdma_process_event(curr_sde,
 500					     sdma_event_e90_sw_halted);
 501		write_sequnlock(&curr_sde->head_lock);
 502		spin_unlock_irqrestore(&curr_sde->tail_lock, flags);
 503	}
 504	schedule_work(&sde->err_halt_worker);
 505}
 506
 507static void sdma_hw_clean_up_task(struct tasklet_struct *t)
 508{
 509	struct sdma_engine *sde = from_tasklet(sde, t,
 510					       sdma_hw_clean_up_task);
 511	u64 statuscsr;
 512
 513	while (1) {
 514#ifdef CONFIG_SDMA_VERBOSITY
 515		dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
 516			   sde->this_idx, slashstrip(__FILE__), __LINE__,
 517			__func__);
 518#endif
 519		statuscsr = read_sde_csr(sde, SD(STATUS));
 520		statuscsr &= SD(STATUS_ENG_CLEANED_UP_SMASK);
 521		if (statuscsr)
 522			break;
 523		udelay(10);
 524	}
 525
 526	sdma_process_event(sde, sdma_event_e25_hw_clean_up_done);
 527}
 528
 529static inline struct sdma_txreq *get_txhead(struct sdma_engine *sde)
 530{
 531	return sde->tx_ring[sde->tx_head & sde->sdma_mask];
 532}
 533
 534/*
 535 * flush ring for recovery
 536 */
 537static void sdma_flush_descq(struct sdma_engine *sde)
 538{
 539	u16 head, tail;
 540	int progress = 0;
 541	struct sdma_txreq *txp = get_txhead(sde);
 542
 543	/* The reason for some of the complexity of this code is that
 544	 * not all descriptors have corresponding txps.  So, we have to
 545	 * be able to skip over descs until we wander into the range of
 546	 * the next txp on the list.
 547	 */
 548	head = sde->descq_head & sde->sdma_mask;
 549	tail = sde->descq_tail & sde->sdma_mask;
 550	while (head != tail) {
 551		/* advance head, wrap if needed */
 552		head = ++sde->descq_head & sde->sdma_mask;
 553		/* if now past this txp's descs, do the callback */
 554		if (txp && txp->next_descq_idx == head) {
 555			/* remove from list */
 556			sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL;
 557			complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED);
 558			trace_hfi1_sdma_progress(sde, head, tail, txp);
 559			txp = get_txhead(sde);
 560		}
 561		progress++;
 562	}
 563	if (progress)
 564		sdma_desc_avail(sde, sdma_descq_freecnt(sde));
 565}
 566
 567static void sdma_sw_clean_up_task(struct tasklet_struct *t)
 568{
 569	struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task);
 570	unsigned long flags;
 571
 572	spin_lock_irqsave(&sde->tail_lock, flags);
 573	write_seqlock(&sde->head_lock);
 574
 575	/*
 576	 * At this point, the following should always be true:
 577	 * - We are halted, so no more descriptors are getting retired.
 578	 * - We are not running, so no one is submitting new work.
 579	 * - Only we can send the e40_sw_cleaned, so we can't start
 580	 *   running again until we say so.  So, the active list and
 581	 *   descq are ours to play with.
 582	 */
 583
 584	/*
 585	 * In the error clean up sequence, software clean must be called
 586	 * before the hardware clean so we can use the hardware head in
 587	 * the progress routine.  A hardware clean or SPC unfreeze will
 588	 * reset the hardware head.
 589	 *
 590	 * Process all retired requests. The progress routine will use the
 591	 * latest physical hardware head - we are not running so speed does
 592	 * not matter.
 593	 */
 594	sdma_make_progress(sde, 0);
 595
 596	sdma_flush(sde);
 597
 598	/*
 599	 * Reset our notion of head and tail.
 600	 * Note that the HW registers have been reset via an earlier
 601	 * clean up.
 602	 */
 603	sde->descq_tail = 0;
 604	sde->descq_head = 0;
 605	sde->desc_avail = sdma_descq_freecnt(sde);
 606	*sde->head_dma = 0;
 607
 608	__sdma_process_event(sde, sdma_event_e40_sw_cleaned);
 609
 610	write_sequnlock(&sde->head_lock);
 611	spin_unlock_irqrestore(&sde->tail_lock, flags);
 612}
 613
 614static void sdma_sw_tear_down(struct sdma_engine *sde)
 615{
 616	struct sdma_state *ss = &sde->state;
 617
 618	/* Releasing this reference means the state machine has stopped. */
 619	sdma_put(ss);
 620
 621	/* stop waiting for all unfreeze events to complete */
 622	atomic_set(&sde->dd->sdma_unfreeze_count, -1);
 623	wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
 624}
 625
 626static void sdma_start_hw_clean_up(struct sdma_engine *sde)
 627{
 628	tasklet_hi_schedule(&sde->sdma_hw_clean_up_task);
 629}
 630
 631static void sdma_set_state(struct sdma_engine *sde,
 632			   enum sdma_states next_state)
 633{
 634	struct sdma_state *ss = &sde->state;
 635	const struct sdma_set_state_action *action = sdma_action_table;
 636	unsigned op = 0;
 637
 638	trace_hfi1_sdma_state(
 639		sde,
 640		sdma_state_names[ss->current_state],
 641		sdma_state_names[next_state]);
 642
 643	/* debugging bookkeeping */
 644	ss->previous_state = ss->current_state;
 645	ss->previous_op = ss->current_op;
 646	ss->current_state = next_state;
 647
 648	if (ss->previous_state != sdma_state_s99_running &&
 649	    next_state == sdma_state_s99_running)
 650		sdma_flush(sde);
 651
 652	if (action[next_state].op_enable)
 653		op |= SDMA_SENDCTRL_OP_ENABLE;
 654
 655	if (action[next_state].op_intenable)
 656		op |= SDMA_SENDCTRL_OP_INTENABLE;
 657
 658	if (action[next_state].op_halt)
 659		op |= SDMA_SENDCTRL_OP_HALT;
 660
 661	if (action[next_state].op_cleanup)
 662		op |= SDMA_SENDCTRL_OP_CLEANUP;
 663
 664	if (action[next_state].go_s99_running_tofalse)
 665		ss->go_s99_running = 0;
 666
 667	if (action[next_state].go_s99_running_totrue)
 668		ss->go_s99_running = 1;
 669
 670	ss->current_op = op;
 671	sdma_sendctrl(sde, ss->current_op);
 672}
 673
 674/**
 675 * sdma_get_descq_cnt() - called when device probed
 676 *
 677 * Return a validated descq count.
 678 *
 679 * This is currently only used in the verbs initialization to build the tx
 680 * list.
 681 *
 682 * This will probably be deleted in favor of a more scalable approach to
 683 * alloc tx's.
 684 *
 685 */
 686u16 sdma_get_descq_cnt(void)
 687{
 688	u16 count = sdma_descq_cnt;
 689
 690	if (!count)
 691		return SDMA_DESCQ_CNT;
 692	/* count must be a power of 2 greater than 64 and less than
 693	 * 32768.   Otherwise return default.
 694	 */
 695	if (!is_power_of_2(count))
 696		return SDMA_DESCQ_CNT;
 697	if (count < 64 || count > 32768)
 698		return SDMA_DESCQ_CNT;
 699	return count;
 700}
 701
 702/**
 703 * sdma_engine_get_vl() - return vl for a given sdma engine
 704 * @sde: sdma engine
 705 *
 706 * This function returns the vl mapped to a given engine, or an error if
 707 * the mapping can't be found. The mapping fields are protected by RCU.
 708 */
 709int sdma_engine_get_vl(struct sdma_engine *sde)
 710{
 711	struct hfi1_devdata *dd = sde->dd;
 712	struct sdma_vl_map *m;
 713	u8 vl;
 714
 715	if (sde->this_idx >= TXE_NUM_SDMA_ENGINES)
 716		return -EINVAL;
 717
 718	rcu_read_lock();
 719	m = rcu_dereference(dd->sdma_map);
 720	if (unlikely(!m)) {
 721		rcu_read_unlock();
 722		return -EINVAL;
 723	}
 724	vl = m->engine_to_vl[sde->this_idx];
 725	rcu_read_unlock();
 726
 727	return vl;
 728}
 729
 730/**
 731 * sdma_select_engine_vl() - select sdma engine
 732 * @dd: devdata
 733 * @selector: a spreading factor
 734 * @vl: this vl
 735 *
 736 *
 737 * This function returns an engine based on the selector and a vl.  The
 738 * mapping fields are protected by RCU.
 739 */
 740struct sdma_engine *sdma_select_engine_vl(
 741	struct hfi1_devdata *dd,
 742	u32 selector,
 743	u8 vl)
 744{
 745	struct sdma_vl_map *m;
 746	struct sdma_map_elem *e;
 747	struct sdma_engine *rval;
 748
 749	/* NOTE This should only happen if SC->VL changed after the initial
 750	 *      checks on the QP/AH
 751	 *      Default will return engine 0 below
 752	 */
 753	if (vl >= num_vls) {
 754		rval = NULL;
 755		goto done;
 756	}
 757
 758	rcu_read_lock();
 759	m = rcu_dereference(dd->sdma_map);
 760	if (unlikely(!m)) {
 761		rcu_read_unlock();
 762		return &dd->per_sdma[0];
 763	}
 764	e = m->map[vl & m->mask];
 765	rval = e->sde[selector & e->mask];
 766	rcu_read_unlock();
 767
 768done:
 769	rval =  !rval ? &dd->per_sdma[0] : rval;
 770	trace_hfi1_sdma_engine_select(dd, selector, vl, rval->this_idx);
 771	return rval;
 772}
 773
 774/**
 775 * sdma_select_engine_sc() - select sdma engine
 776 * @dd: devdata
 777 * @selector: a spreading factor
 778 * @sc5: the 5 bit sc
 779 *
 780 *
 781 * This function returns an engine based on the selector and an sc.
 782 */
 783struct sdma_engine *sdma_select_engine_sc(
 784	struct hfi1_devdata *dd,
 785	u32 selector,
 786	u8 sc5)
 787{
 788	u8 vl = sc_to_vlt(dd, sc5);
 789
 790	return sdma_select_engine_vl(dd, selector, vl);
 791}
 792
 793struct sdma_rht_map_elem {
 794	u32 mask;
 795	u8 ctr;
 796	struct sdma_engine *sde[];
 797};
 798
 799struct sdma_rht_node {
 800	unsigned long cpu_id;
 801	struct sdma_rht_map_elem *map[HFI1_MAX_VLS_SUPPORTED];
 802	struct rhash_head node;
 803};
 804
 805#define NR_CPUS_HINT 192
 806
 807static const struct rhashtable_params sdma_rht_params = {
 808	.nelem_hint = NR_CPUS_HINT,
 809	.head_offset = offsetof(struct sdma_rht_node, node),
 810	.key_offset = offsetof(struct sdma_rht_node, cpu_id),
 811	.key_len = sizeof_field(struct sdma_rht_node, cpu_id),
 812	.max_size = NR_CPUS,
 813	.min_size = 8,
 814	.automatic_shrinking = true,
 815};
 816
 817/*
 818 * sdma_select_user_engine() - select sdma engine based on user setup
 819 * @dd: devdata
 820 * @selector: a spreading factor
 821 * @vl: this vl
 822 *
 823 * This function returns an sdma engine for a user sdma request.
 824 * User defined sdma engine affinity setting is honored when applicable,
 825 * otherwise system default sdma engine mapping is used. To ensure correct
 826 * ordering, the mapping from <selector, vl> to sde must remain unchanged.
 827 */
 828struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd,
 829					    u32 selector, u8 vl)
 830{
 831	struct sdma_rht_node *rht_node;
 832	struct sdma_engine *sde = NULL;
 833	unsigned long cpu_id;
 834
 835	/*
 836	 * To ensure that always the same sdma engine(s) will be
 837	 * selected make sure the process is pinned to this CPU only.
 838	 */
 839	if (current->nr_cpus_allowed != 1)
 840		goto out;
 841
 842	rcu_read_lock();
 843	cpu_id = smp_processor_id();
 844	rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id,
 845				     sdma_rht_params);
 846
 847	if (rht_node && rht_node->map[vl]) {
 848		struct sdma_rht_map_elem *map = rht_node->map[vl];
 849
 850		sde = map->sde[selector & map->mask];
 851	}
 852	rcu_read_unlock();
 853
 854	if (sde)
 855		return sde;
 856
 857out:
 858	return sdma_select_engine_vl(dd, selector, vl);
 859}
 860
 861static void sdma_populate_sde_map(struct sdma_rht_map_elem *map)
 862{
 863	int i;
 864
 865	for (i = 0; i < roundup_pow_of_two(map->ctr ? : 1) - map->ctr; i++)
 866		map->sde[map->ctr + i] = map->sde[i];
 867}
 868
 869static void sdma_cleanup_sde_map(struct sdma_rht_map_elem *map,
 870				 struct sdma_engine *sde)
 871{
 872	unsigned int i, pow;
 873
 874	/* only need to check the first ctr entries for a match */
 875	for (i = 0; i < map->ctr; i++) {
 876		if (map->sde[i] == sde) {
 877			memmove(&map->sde[i], &map->sde[i + 1],
 878				(map->ctr - i - 1) * sizeof(map->sde[0]));
 879			map->ctr--;
 880			pow = roundup_pow_of_two(map->ctr ? : 1);
 881			map->mask = pow - 1;
 882			sdma_populate_sde_map(map);
 883			break;
 884		}
 885	}
 886}
 887
 888/*
 889 * Prevents concurrent reads and writes of the sdma engine cpu_mask
 890 */
 891static DEFINE_MUTEX(process_to_sde_mutex);
 892
 893ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf,
 894				size_t count)
 895{
 896	struct hfi1_devdata *dd = sde->dd;
 897	cpumask_var_t mask, new_mask;
 898	unsigned long cpu;
 899	int ret, vl, sz;
 900	struct sdma_rht_node *rht_node;
 901
 902	vl = sdma_engine_get_vl(sde);
 903	if (unlikely(vl < 0 || vl >= ARRAY_SIZE(rht_node->map)))
 904		return -EINVAL;
 905
 906	ret = zalloc_cpumask_var(&mask, GFP_KERNEL);
 907	if (!ret)
 908		return -ENOMEM;
 909
 910	ret = zalloc_cpumask_var(&new_mask, GFP_KERNEL);
 911	if (!ret) {
 912		free_cpumask_var(mask);
 913		return -ENOMEM;
 914	}
 915	ret = cpulist_parse(buf, mask);
 916	if (ret)
 917		goto out_free;
 918
 919	if (!cpumask_subset(mask, cpu_online_mask)) {
 920		dd_dev_warn(sde->dd, "Invalid CPU mask\n");
 921		ret = -EINVAL;
 922		goto out_free;
 923	}
 924
 925	sz = sizeof(struct sdma_rht_map_elem) +
 926			(TXE_NUM_SDMA_ENGINES * sizeof(struct sdma_engine *));
 927
 928	mutex_lock(&process_to_sde_mutex);
 929
 930	for_each_cpu(cpu, mask) {
 931		/* Check if we have this already mapped */
 932		if (cpumask_test_cpu(cpu, &sde->cpu_mask)) {
 933			cpumask_set_cpu(cpu, new_mask);
 934			continue;
 935		}
 936
 937		rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu,
 938						  sdma_rht_params);
 939		if (!rht_node) {
 940			rht_node = kzalloc(sizeof(*rht_node), GFP_KERNEL);
 941			if (!rht_node) {
 942				ret = -ENOMEM;
 943				goto out;
 944			}
 945
 946			rht_node->map[vl] = kzalloc(sz, GFP_KERNEL);
 947			if (!rht_node->map[vl]) {
 948				kfree(rht_node);
 949				ret = -ENOMEM;
 950				goto out;
 951			}
 952			rht_node->cpu_id = cpu;
 953			rht_node->map[vl]->mask = 0;
 954			rht_node->map[vl]->ctr = 1;
 955			rht_node->map[vl]->sde[0] = sde;
 956
 957			ret = rhashtable_insert_fast(dd->sdma_rht,
 958						     &rht_node->node,
 959						     sdma_rht_params);
 960			if (ret) {
 961				kfree(rht_node->map[vl]);
 962				kfree(rht_node);
 963				dd_dev_err(sde->dd, "Failed to set process to sde affinity for cpu %lu\n",
 964					   cpu);
 965				goto out;
 966			}
 967
 968		} else {
 969			int ctr, pow;
 970
 971			/* Add new user mappings */
 972			if (!rht_node->map[vl])
 973				rht_node->map[vl] = kzalloc(sz, GFP_KERNEL);
 974
 975			if (!rht_node->map[vl]) {
 976				ret = -ENOMEM;
 977				goto out;
 978			}
 979
 980			rht_node->map[vl]->ctr++;
 981			ctr = rht_node->map[vl]->ctr;
 982			rht_node->map[vl]->sde[ctr - 1] = sde;
 983			pow = roundup_pow_of_two(ctr);
 984			rht_node->map[vl]->mask = pow - 1;
 985
 986			/* Populate the sde map table */
 987			sdma_populate_sde_map(rht_node->map[vl]);
 988		}
 989		cpumask_set_cpu(cpu, new_mask);
 990	}
 991
 992	/* Clean up old mappings */
 993	for_each_online_cpu(cpu) {
 994		struct sdma_rht_node *rht_node;
 995
 996		/* Don't cleanup sdes that are set in the new mask */
 997		if (cpumask_test_cpu(cpu, mask))
 998			continue;
 999
1000		rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu,
1001						  sdma_rht_params);
1002		if (rht_node) {
1003			bool empty = true;
1004			int i;
1005
1006			/* Remove mappings for old sde */
1007			for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
1008				if (rht_node->map[i])
1009					sdma_cleanup_sde_map(rht_node->map[i],
1010							     sde);
1011
1012			/* Free empty hash table entries */
1013			for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) {
1014				if (!rht_node->map[i])
1015					continue;
1016
1017				if (rht_node->map[i]->ctr) {
1018					empty = false;
1019					break;
1020				}
1021			}
1022
1023			if (empty) {
1024				ret = rhashtable_remove_fast(dd->sdma_rht,
1025							     &rht_node->node,
1026							     sdma_rht_params);
1027				WARN_ON(ret);
1028
1029				for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
1030					kfree(rht_node->map[i]);
1031
1032				kfree(rht_node);
1033			}
1034		}
1035	}
1036
1037	cpumask_copy(&sde->cpu_mask, new_mask);
1038out:
1039	mutex_unlock(&process_to_sde_mutex);
1040out_free:
1041	free_cpumask_var(mask);
1042	free_cpumask_var(new_mask);
1043	return ret ? : strnlen(buf, PAGE_SIZE);
1044}
1045
1046ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf)
1047{
1048	mutex_lock(&process_to_sde_mutex);
1049	if (cpumask_empty(&sde->cpu_mask))
1050		snprintf(buf, PAGE_SIZE, "%s\n", "empty");
1051	else
1052		cpumap_print_to_pagebuf(true, buf, &sde->cpu_mask);
1053	mutex_unlock(&process_to_sde_mutex);
1054	return strnlen(buf, PAGE_SIZE);
1055}
1056
1057static void sdma_rht_free(void *ptr, void *arg)
1058{
1059	struct sdma_rht_node *rht_node = ptr;
1060	int i;
1061
1062	for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
1063		kfree(rht_node->map[i]);
1064
1065	kfree(rht_node);
1066}
1067
1068/**
1069 * sdma_seqfile_dump_cpu_list() - debugfs dump the cpu to sdma mappings
1070 * @s: seq file
1071 * @dd: hfi1_devdata
1072 * @cpuid: cpu id
1073 *
1074 * This routine dumps the process to sde mappings per cpu
1075 */
1076void sdma_seqfile_dump_cpu_list(struct seq_file *s,
1077				struct hfi1_devdata *dd,
1078				unsigned long cpuid)
1079{
1080	struct sdma_rht_node *rht_node;
1081	int i, j;
1082
1083	rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpuid,
1084					  sdma_rht_params);
1085	if (!rht_node)
1086		return;
1087
1088	seq_printf(s, "cpu%3lu: ", cpuid);
1089	for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) {
1090		if (!rht_node->map[i] || !rht_node->map[i]->ctr)
1091			continue;
1092
1093		seq_printf(s, " vl%d: [", i);
1094
1095		for (j = 0; j < rht_node->map[i]->ctr; j++) {
1096			if (!rht_node->map[i]->sde[j])
1097				continue;
1098
1099			if (j > 0)
1100				seq_puts(s, ",");
1101
1102			seq_printf(s, " sdma%2d",
1103				   rht_node->map[i]->sde[j]->this_idx);
1104		}
1105		seq_puts(s, " ]");
1106	}
1107
1108	seq_puts(s, "\n");
1109}
1110
1111/*
1112 * Free the indicated map struct
1113 */
1114static void sdma_map_free(struct sdma_vl_map *m)
1115{
1116	int i;
1117
1118	for (i = 0; m && i < m->actual_vls; i++)
1119		kfree(m->map[i]);
1120	kfree(m);
1121}
1122
1123/*
1124 * Handle RCU callback
1125 */
1126static void sdma_map_rcu_callback(struct rcu_head *list)
1127{
1128	struct sdma_vl_map *m = container_of(list, struct sdma_vl_map, list);
1129
1130	sdma_map_free(m);
1131}
1132
1133/**
1134 * sdma_map_init - called when # vls change
1135 * @dd: hfi1_devdata
1136 * @port: port number
1137 * @num_vls: number of vls
1138 * @vl_engines: per vl engine mapping (optional)
1139 *
1140 * This routine changes the mapping based on the number of vls.
1141 *
1142 * vl_engines is used to specify a non-uniform vl/engine loading. NULL
1143 * implies auto computing the loading and giving each VLs a uniform
1144 * distribution of engines per VL.
1145 *
1146 * The auto algorithm computes the sde_per_vl and the number of extra
1147 * engines.  Any extra engines are added from the last VL on down.
1148 *
1149 * rcu locking is used here to control access to the mapping fields.
1150 *
1151 * If either the num_vls or num_sdma are non-power of 2, the array sizes
1152 * in the struct sdma_vl_map and the struct sdma_map_elem are rounded
1153 * up to the next highest power of 2 and the first entry is reused
1154 * in a round robin fashion.
1155 *
1156 * If an error occurs the map change is not done and the mapping is
1157 * not changed.
1158 *
1159 */
1160int sdma_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_engines)
1161{
1162	int i, j;
1163	int extra, sde_per_vl;
1164	int engine = 0;
1165	u8 lvl_engines[OPA_MAX_VLS];
1166	struct sdma_vl_map *oldmap, *newmap;
1167
1168	if (!(dd->flags & HFI1_HAS_SEND_DMA))
1169		return 0;
1170
1171	if (!vl_engines) {
1172		/* truncate divide */
1173		sde_per_vl = dd->num_sdma / num_vls;
1174		/* extras */
1175		extra = dd->num_sdma % num_vls;
1176		vl_engines = lvl_engines;
1177		/* add extras from last vl down */
1178		for (i = num_vls - 1; i >= 0; i--, extra--)
1179			vl_engines[i] = sde_per_vl + (extra > 0 ? 1 : 0);
1180	}
1181	/* build new map */
1182	newmap = kzalloc(
1183		sizeof(struct sdma_vl_map) +
1184			roundup_pow_of_two(num_vls) *
1185			sizeof(struct sdma_map_elem *),
1186		GFP_KERNEL);
1187	if (!newmap)
1188		goto bail;
1189	newmap->actual_vls = num_vls;
1190	newmap->vls = roundup_pow_of_two(num_vls);
1191	newmap->mask = (1 << ilog2(newmap->vls)) - 1;
1192	/* initialize back-map */
1193	for (i = 0; i < TXE_NUM_SDMA_ENGINES; i++)
1194		newmap->engine_to_vl[i] = -1;
1195	for (i = 0; i < newmap->vls; i++) {
1196		/* save for wrap around */
1197		int first_engine = engine;
1198
1199		if (i < newmap->actual_vls) {
1200			int sz = roundup_pow_of_two(vl_engines[i]);
1201
1202			/* only allocate once */
1203			newmap->map[i] = kzalloc(
1204				sizeof(struct sdma_map_elem) +
1205					sz * sizeof(struct sdma_engine *),
1206				GFP_KERNEL);
1207			if (!newmap->map[i])
1208				goto bail;
1209			newmap->map[i]->mask = (1 << ilog2(sz)) - 1;
1210			/* assign engines */
1211			for (j = 0; j < sz; j++) {
1212				newmap->map[i]->sde[j] =
1213					&dd->per_sdma[engine];
1214				if (++engine >= first_engine + vl_engines[i])
1215					/* wrap back to first engine */
1216					engine = first_engine;
1217			}
1218			/* assign back-map */
1219			for (j = 0; j < vl_engines[i]; j++)
1220				newmap->engine_to_vl[first_engine + j] = i;
1221		} else {
1222			/* just re-use entry without allocating */
1223			newmap->map[i] = newmap->map[i % num_vls];
1224		}
1225		engine = first_engine + vl_engines[i];
1226	}
1227	/* newmap in hand, save old map */
1228	spin_lock_irq(&dd->sde_map_lock);
1229	oldmap = rcu_dereference_protected(dd->sdma_map,
1230					   lockdep_is_held(&dd->sde_map_lock));
1231
1232	/* publish newmap */
1233	rcu_assign_pointer(dd->sdma_map, newmap);
1234
1235	spin_unlock_irq(&dd->sde_map_lock);
1236	/* success, free any old map after grace period */
1237	if (oldmap)
1238		call_rcu(&oldmap->list, sdma_map_rcu_callback);
1239	return 0;
1240bail:
1241	/* free any partial allocation */
1242	sdma_map_free(newmap);
1243	return -ENOMEM;
1244}
1245
1246/**
1247 * sdma_clean - Clean up allocated memory
1248 * @dd:          struct hfi1_devdata
1249 * @num_engines: num sdma engines
1250 *
1251 * This routine can be called regardless of the success of
1252 * sdma_init()
1253 */
1254void sdma_clean(struct hfi1_devdata *dd, size_t num_engines)
1255{
1256	size_t i;
1257	struct sdma_engine *sde;
1258
1259	if (dd->sdma_pad_dma) {
1260		dma_free_coherent(&dd->pcidev->dev, SDMA_PAD,
1261				  (void *)dd->sdma_pad_dma,
1262				  dd->sdma_pad_phys);
1263		dd->sdma_pad_dma = NULL;
1264		dd->sdma_pad_phys = 0;
1265	}
1266	if (dd->sdma_heads_dma) {
1267		dma_free_coherent(&dd->pcidev->dev, dd->sdma_heads_size,
1268				  (void *)dd->sdma_heads_dma,
1269				  dd->sdma_heads_phys);
1270		dd->sdma_heads_dma = NULL;
1271		dd->sdma_heads_phys = 0;
1272	}
1273	for (i = 0; dd->per_sdma && i < num_engines; ++i) {
1274		sde = &dd->per_sdma[i];
1275
1276		sde->head_dma = NULL;
1277		sde->head_phys = 0;
1278
1279		if (sde->descq) {
1280			dma_free_coherent(
1281				&dd->pcidev->dev,
1282				sde->descq_cnt * sizeof(u64[2]),
1283				sde->descq,
1284				sde->descq_phys
1285			);
1286			sde->descq = NULL;
1287			sde->descq_phys = 0;
1288		}
1289		kvfree(sde->tx_ring);
1290		sde->tx_ring = NULL;
1291	}
1292	if (rcu_access_pointer(dd->sdma_map)) {
1293		spin_lock_irq(&dd->sde_map_lock);
1294		sdma_map_free(rcu_access_pointer(dd->sdma_map));
1295		RCU_INIT_POINTER(dd->sdma_map, NULL);
1296		spin_unlock_irq(&dd->sde_map_lock);
1297		synchronize_rcu();
1298	}
1299	kfree(dd->per_sdma);
1300	dd->per_sdma = NULL;
1301
1302	if (dd->sdma_rht) {
1303		rhashtable_free_and_destroy(dd->sdma_rht, sdma_rht_free, NULL);
1304		kfree(dd->sdma_rht);
1305		dd->sdma_rht = NULL;
1306	}
1307}
1308
1309/**
1310 * sdma_init() - called when device probed
1311 * @dd: hfi1_devdata
1312 * @port: port number (currently only zero)
1313 *
1314 * Initializes each sde and its csrs.
1315 * Interrupts are not required to be enabled.
1316 *
1317 * Returns:
1318 * 0 - success, -errno on failure
1319 */
1320int sdma_init(struct hfi1_devdata *dd, u8 port)
1321{
1322	unsigned this_idx;
1323	struct sdma_engine *sde;
1324	struct rhashtable *tmp_sdma_rht;
1325	u16 descq_cnt;
1326	void *curr_head;
1327	struct hfi1_pportdata *ppd = dd->pport + port;
1328	u32 per_sdma_credits;
1329	uint idle_cnt = sdma_idle_cnt;
1330	size_t num_engines = chip_sdma_engines(dd);
1331	int ret = -ENOMEM;
1332
1333	if (!HFI1_CAP_IS_KSET(SDMA)) {
1334		HFI1_CAP_CLEAR(SDMA_AHG);
1335		return 0;
1336	}
1337	if (mod_num_sdma &&
1338	    /* can't exceed chip support */
1339	    mod_num_sdma <= chip_sdma_engines(dd) &&
1340	    /* count must be >= vls */
1341	    mod_num_sdma >= num_vls)
1342		num_engines = mod_num_sdma;
1343
1344	dd_dev_info(dd, "SDMA mod_num_sdma: %u\n", mod_num_sdma);
1345	dd_dev_info(dd, "SDMA chip_sdma_engines: %u\n", chip_sdma_engines(dd));
1346	dd_dev_info(dd, "SDMA chip_sdma_mem_size: %u\n",
1347		    chip_sdma_mem_size(dd));
1348
1349	per_sdma_credits =
1350		chip_sdma_mem_size(dd) / (num_engines * SDMA_BLOCK_SIZE);
1351
1352	/* set up freeze waitqueue */
1353	init_waitqueue_head(&dd->sdma_unfreeze_wq);
1354	atomic_set(&dd->sdma_unfreeze_count, 0);
1355
1356	descq_cnt = sdma_get_descq_cnt();
1357	dd_dev_info(dd, "SDMA engines %zu descq_cnt %u\n",
1358		    num_engines, descq_cnt);
1359
1360	/* alloc memory for array of send engines */
1361	dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma),
1362				    GFP_KERNEL, dd->node);
1363	if (!dd->per_sdma)
1364		return ret;
1365
1366	idle_cnt = ns_to_cclock(dd, idle_cnt);
1367	if (idle_cnt)
1368		dd->default_desc1 =
1369			SDMA_DESC1_HEAD_TO_HOST_FLAG;
1370	else
1371		dd->default_desc1 =
1372			SDMA_DESC1_INT_REQ_FLAG;
1373
1374	if (!sdma_desct_intr)
1375		sdma_desct_intr = SDMA_DESC_INTR;
1376
1377	/* Allocate memory for SendDMA descriptor FIFOs */
1378	for (this_idx = 0; this_idx < num_engines; ++this_idx) {
1379		sde = &dd->per_sdma[this_idx];
1380		sde->dd = dd;
1381		sde->ppd = ppd;
1382		sde->this_idx = this_idx;
1383		sde->descq_cnt = descq_cnt;
1384		sde->desc_avail = sdma_descq_freecnt(sde);
1385		sde->sdma_shift = ilog2(descq_cnt);
1386		sde->sdma_mask = (1 << sde->sdma_shift) - 1;
1387
1388		/* Create a mask specifically for each interrupt source */
1389		sde->int_mask = (u64)1 << (0 * TXE_NUM_SDMA_ENGINES +
1390					   this_idx);
1391		sde->progress_mask = (u64)1 << (1 * TXE_NUM_SDMA_ENGINES +
1392						this_idx);
1393		sde->idle_mask = (u64)1 << (2 * TXE_NUM_SDMA_ENGINES +
1394					    this_idx);
1395		/* Create a combined mask to cover all 3 interrupt sources */
1396		sde->imask = sde->int_mask | sde->progress_mask |
1397			     sde->idle_mask;
1398
1399		spin_lock_init(&sde->tail_lock);
1400		seqlock_init(&sde->head_lock);
1401		spin_lock_init(&sde->senddmactrl_lock);
1402		spin_lock_init(&sde->flushlist_lock);
1403		seqlock_init(&sde->waitlock);
1404		/* insure there is always a zero bit */
1405		sde->ahg_bits = 0xfffffffe00000000ULL;
1406
1407		sdma_set_state(sde, sdma_state_s00_hw_down);
1408
1409		/* set up reference counting */
1410		kref_init(&sde->state.kref);
1411		init_completion(&sde->state.comp);
1412
1413		INIT_LIST_HEAD(&sde->flushlist);
1414		INIT_LIST_HEAD(&sde->dmawait);
1415
1416		sde->tail_csr =
1417			get_kctxt_csr_addr(dd, this_idx, SD(TAIL));
1418
1419		tasklet_setup(&sde->sdma_hw_clean_up_task,
1420			      sdma_hw_clean_up_task);
1421		tasklet_setup(&sde->sdma_sw_clean_up_task,
1422			      sdma_sw_clean_up_task);
1423		INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait);
1424		INIT_WORK(&sde->flush_worker, sdma_field_flush);
1425
1426		sde->progress_check_head = 0;
1427
1428		timer_setup(&sde->err_progress_check_timer,
1429			    sdma_err_progress_check, 0);
1430
1431		sde->descq = dma_alloc_coherent(&dd->pcidev->dev,
1432						descq_cnt * sizeof(u64[2]),
1433						&sde->descq_phys, GFP_KERNEL);
1434		if (!sde->descq)
1435			goto bail;
1436		sde->tx_ring =
1437			kvzalloc_node(array_size(descq_cnt,
1438						 sizeof(struct sdma_txreq *)),
1439				      GFP_KERNEL, dd->node);
1440		if (!sde->tx_ring)
1441			goto bail;
1442	}
1443
1444	dd->sdma_heads_size = L1_CACHE_BYTES * num_engines;
1445	/* Allocate memory for DMA of head registers to memory */
1446	dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev,
1447						dd->sdma_heads_size,
1448						&dd->sdma_heads_phys,
1449						GFP_KERNEL);
1450	if (!dd->sdma_heads_dma) {
1451		dd_dev_err(dd, "failed to allocate SendDMA head memory\n");
1452		goto bail;
1453	}
1454
1455	/* Allocate memory for pad */
1456	dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD,
1457					      &dd->sdma_pad_phys, GFP_KERNEL);
1458	if (!dd->sdma_pad_dma) {
1459		dd_dev_err(dd, "failed to allocate SendDMA pad memory\n");
1460		goto bail;
1461	}
1462
1463	/* assign each engine to different cacheline and init registers */
1464	curr_head = (void *)dd->sdma_heads_dma;
1465	for (this_idx = 0; this_idx < num_engines; ++this_idx) {
1466		unsigned long phys_offset;
1467
1468		sde = &dd->per_sdma[this_idx];
1469
1470		sde->head_dma = curr_head;
1471		curr_head += L1_CACHE_BYTES;
1472		phys_offset = (unsigned long)sde->head_dma -
1473			      (unsigned long)dd->sdma_heads_dma;
1474		sde->head_phys = dd->sdma_heads_phys + phys_offset;
1475		init_sdma_regs(sde, per_sdma_credits, idle_cnt);
1476	}
1477	dd->flags |= HFI1_HAS_SEND_DMA;
1478	dd->flags |= idle_cnt ? HFI1_HAS_SDMA_TIMEOUT : 0;
1479	dd->num_sdma = num_engines;
1480	ret = sdma_map_init(dd, port, ppd->vls_operational, NULL);
1481	if (ret < 0)
1482		goto bail;
1483
1484	tmp_sdma_rht = kzalloc(sizeof(*tmp_sdma_rht), GFP_KERNEL);
1485	if (!tmp_sdma_rht) {
1486		ret = -ENOMEM;
1487		goto bail;
1488	}
1489
1490	ret = rhashtable_init(tmp_sdma_rht, &sdma_rht_params);
1491	if (ret < 0) {
1492		kfree(tmp_sdma_rht);
1493		goto bail;
1494	}
1495
1496	dd->sdma_rht = tmp_sdma_rht;
1497
1498	dd_dev_info(dd, "SDMA num_sdma: %u\n", dd->num_sdma);
1499	return 0;
1500
1501bail:
1502	sdma_clean(dd, num_engines);
1503	return ret;
1504}
1505
1506/**
1507 * sdma_all_running() - called when the link goes up
1508 * @dd: hfi1_devdata
1509 *
1510 * This routine moves all engines to the running state.
1511 */
1512void sdma_all_running(struct hfi1_devdata *dd)
1513{
1514	struct sdma_engine *sde;
1515	unsigned int i;
1516
1517	/* move all engines to running */
1518	for (i = 0; i < dd->num_sdma; ++i) {
1519		sde = &dd->per_sdma[i];
1520		sdma_process_event(sde, sdma_event_e30_go_running);
1521	}
1522}
1523
1524/**
1525 * sdma_start() - called to kick off state processing for all engines
1526 * @dd: hfi1_devdata
1527 *
1528 * This routine is for kicking off the state processing for all required
1529 * sdma engines.  Interrupts need to be working at this point.
1530 *
1531 */
1532void sdma_start(struct hfi1_devdata *dd)
1533{
1534	unsigned i;
1535	struct sdma_engine *sde;
1536
1537	/* kick off the engines state processing */
1538	for (i = 0; i < dd->num_sdma; ++i) {
1539		sde = &dd->per_sdma[i];
1540		sdma_process_event(sde, sdma_event_e10_go_hw_start);
1541	}
1542}
1543
1544/**
1545 * sdma_exit() - used when module is removed
1546 * @dd: hfi1_devdata
1547 */
1548void sdma_exit(struct hfi1_devdata *dd)
1549{
1550	unsigned this_idx;
1551	struct sdma_engine *sde;
1552
1553	for (this_idx = 0; dd->per_sdma && this_idx < dd->num_sdma;
1554			++this_idx) {
1555		sde = &dd->per_sdma[this_idx];
1556		if (!list_empty(&sde->dmawait))
1557			dd_dev_err(dd, "sde %u: dmawait list not empty!\n",
1558				   sde->this_idx);
1559		sdma_process_event(sde, sdma_event_e00_go_hw_down);
1560
1561		timer_delete_sync(&sde->err_progress_check_timer);
1562
1563		/*
1564		 * This waits for the state machine to exit so it is not
1565		 * necessary to kill the sdma_sw_clean_up_task to make sure
1566		 * it is not running.
1567		 */
1568		sdma_finalput(&sde->state);
1569	}
1570}
1571
1572/*
1573 * unmap the indicated descriptor
1574 */
1575static inline void sdma_unmap_desc(
1576	struct hfi1_devdata *dd,
1577	struct sdma_desc *descp)
1578{
1579	switch (sdma_mapping_type(descp)) {
1580	case SDMA_MAP_SINGLE:
1581		dma_unmap_single(&dd->pcidev->dev, sdma_mapping_addr(descp),
1582				 sdma_mapping_len(descp), DMA_TO_DEVICE);
1583		break;
1584	case SDMA_MAP_PAGE:
1585		dma_unmap_page(&dd->pcidev->dev, sdma_mapping_addr(descp),
1586			       sdma_mapping_len(descp), DMA_TO_DEVICE);
1587		break;
1588	}
1589
1590	if (descp->pinning_ctx && descp->ctx_put)
1591		descp->ctx_put(descp->pinning_ctx);
1592	descp->pinning_ctx = NULL;
1593}
1594
1595/*
1596 * return the mode as indicated by the first
1597 * descriptor in the tx.
1598 */
1599static inline u8 ahg_mode(struct sdma_txreq *tx)
1600{
1601	return (tx->descp[0].qw[1] & SDMA_DESC1_HEADER_MODE_SMASK)
1602		>> SDMA_DESC1_HEADER_MODE_SHIFT;
1603}
1604
1605/**
1606 * __sdma_txclean() - clean tx of mappings, descp *kmalloc's
1607 * @dd: hfi1_devdata for unmapping
1608 * @tx: tx request to clean
1609 *
1610 * This is used in the progress routine to clean the tx or
1611 * by the ULP to toss an in-process tx build.
1612 *
1613 * The code can be called multiple times without issue.
1614 *
1615 */
1616void __sdma_txclean(
1617	struct hfi1_devdata *dd,
1618	struct sdma_txreq *tx)
1619{
1620	u16 i;
1621
1622	if (tx->num_desc) {
1623		u8 skip = 0, mode = ahg_mode(tx);
1624
1625		/* unmap first */
1626		sdma_unmap_desc(dd, &tx->descp[0]);
1627		/* determine number of AHG descriptors to skip */
1628		if (mode > SDMA_AHG_APPLY_UPDATE1)
1629			skip = mode >> 1;
1630		for (i = 1 + skip; i < tx->num_desc; i++)
1631			sdma_unmap_desc(dd, &tx->descp[i]);
1632		tx->num_desc = 0;
1633	}
1634	kfree(tx->coalesce_buf);
1635	tx->coalesce_buf = NULL;
1636	/* kmalloc'ed descp */
1637	if (unlikely(tx->desc_limit > ARRAY_SIZE(tx->descs))) {
1638		tx->desc_limit = ARRAY_SIZE(tx->descs);
1639		kfree(tx->descp);
1640	}
1641}
1642
1643static inline u16 sdma_gethead(struct sdma_engine *sde)
1644{
1645	struct hfi1_devdata *dd = sde->dd;
1646	int use_dmahead;
1647	u16 hwhead;
1648
1649#ifdef CONFIG_SDMA_VERBOSITY
1650	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
1651		   sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
1652#endif
1653
1654retry:
1655	use_dmahead = HFI1_CAP_IS_KSET(USE_SDMA_HEAD) && __sdma_running(sde) &&
1656					(dd->flags & HFI1_HAS_SDMA_TIMEOUT);
1657	hwhead = use_dmahead ?
1658		(u16)le64_to_cpu(*sde->head_dma) :
1659		(u16)read_sde_csr(sde, SD(HEAD));
1660
1661	if (unlikely(HFI1_CAP_IS_KSET(SDMA_HEAD_CHECK))) {
1662		u16 cnt;
1663		u16 swtail;
1664		u16 swhead;
1665		int sane;
1666
1667		swhead = sde->descq_head & sde->sdma_mask;
1668		/* this code is really bad for cache line trading */
1669		swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask;
1670		cnt = sde->descq_cnt;
1671
1672		if (swhead < swtail)
1673			/* not wrapped */
1674			sane = (hwhead >= swhead) & (hwhead <= swtail);
1675		else if (swhead > swtail)
1676			/* wrapped around */
1677			sane = ((hwhead >= swhead) && (hwhead < cnt)) ||
1678				(hwhead <= swtail);
1679		else
1680			/* empty */
1681			sane = (hwhead == swhead);
1682
1683		if (unlikely(!sane)) {
1684			dd_dev_err(dd, "SDMA(%u) bad head (%s) hwhd=%u swhd=%u swtl=%u cnt=%u\n",
1685				   sde->this_idx,
1686				   use_dmahead ? "dma" : "kreg",
1687				   hwhead, swhead, swtail, cnt);
1688			if (use_dmahead) {
1689				/* try one more time, using csr */
1690				use_dmahead = 0;
1691				goto retry;
1692			}
1693			/* proceed as if no progress */
1694			hwhead = swhead;
1695		}
1696	}
1697	return hwhead;
1698}
1699
1700/*
1701 * This is called when there are send DMA descriptors that might be
1702 * available.
1703 *
1704 * This is called with head_lock held.
1705 */
1706static void sdma_desc_avail(struct sdma_engine *sde, uint avail)
1707{
1708	struct iowait *wait, *nw, *twait;
1709	struct iowait *waits[SDMA_WAIT_BATCH_SIZE];
1710	uint i, n = 0, seq, tidx = 0;
1711
1712#ifdef CONFIG_SDMA_VERBOSITY
1713	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
1714		   slashstrip(__FILE__), __LINE__, __func__);
1715	dd_dev_err(sde->dd, "avail: %u\n", avail);
1716#endif
1717
1718	do {
1719		seq = read_seqbegin(&sde->waitlock);
1720		if (!list_empty(&sde->dmawait)) {
1721			/* at least one item */
1722			write_seqlock(&sde->waitlock);
1723			/* Harvest waiters wanting DMA descriptors */
1724			list_for_each_entry_safe(
1725					wait,
1726					nw,
1727					&sde->dmawait,
1728					list) {
1729				u32 num_desc;
1730
1731				if (!wait->wakeup)
1732					continue;
1733				if (n == ARRAY_SIZE(waits))
1734					break;
1735				iowait_init_priority(wait);
1736				num_desc = iowait_get_all_desc(wait);
1737				if (num_desc > avail)
1738					break;
1739				avail -= num_desc;
1740				/* Find the top-priority wait memeber */
1741				if (n) {
1742					twait = waits[tidx];
1743					tidx =
1744					    iowait_priority_update_top(wait,
1745								       twait,
1746								       n,
1747								       tidx);
1748				}
1749				list_del_init(&wait->list);
1750				waits[n++] = wait;
1751			}
1752			write_sequnlock(&sde->waitlock);
1753			break;
1754		}
1755	} while (read_seqretry(&sde->waitlock, seq));
1756
1757	/* Schedule the top-priority entry first */
1758	if (n)
1759		waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON);
1760
1761	for (i = 0; i < n; i++)
1762		if (i != tidx)
1763			waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON);
1764}
1765
1766/* head_lock must be held */
1767static void sdma_make_progress(struct sdma_engine *sde, u64 status)
1768{
1769	struct sdma_txreq *txp = NULL;
1770	int progress = 0;
1771	u16 hwhead, swhead;
1772	int idle_check_done = 0;
1773
1774	hwhead = sdma_gethead(sde);
1775
1776	/* The reason for some of the complexity of this code is that
1777	 * not all descriptors have corresponding txps.  So, we have to
1778	 * be able to skip over descs until we wander into the range of
1779	 * the next txp on the list.
1780	 */
1781
1782retry:
1783	txp = get_txhead(sde);
1784	swhead = sde->descq_head & sde->sdma_mask;
1785	trace_hfi1_sdma_progress(sde, hwhead, swhead, txp);
1786	while (swhead != hwhead) {
1787		/* advance head, wrap if needed */
1788		swhead = ++sde->descq_head & sde->sdma_mask;
1789
1790		/* if now past this txp's descs, do the callback */
1791		if (txp && txp->next_descq_idx == swhead) {
1792			/* remove from list */
1793			sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL;
1794			complete_tx(sde, txp, SDMA_TXREQ_S_OK);
1795			/* see if there is another txp */
1796			txp = get_txhead(sde);
1797		}
1798		trace_hfi1_sdma_progress(sde, hwhead, swhead, txp);
1799		progress++;
1800	}
1801
1802	/*
1803	 * The SDMA idle interrupt is not guaranteed to be ordered with respect
1804	 * to updates to the dma_head location in host memory. The head
1805	 * value read might not be fully up to date. If there are pending
1806	 * descriptors and the SDMA idle interrupt fired then read from the
1807	 * CSR SDMA head instead to get the latest value from the hardware.
1808	 * The hardware SDMA head should be read at most once in this invocation
1809	 * of sdma_make_progress(..) which is ensured by idle_check_done flag
1810	 */
1811	if ((status & sde->idle_mask) && !idle_check_done) {
1812		u16 swtail;
1813
1814		swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask;
1815		if (swtail != hwhead) {
1816			hwhead = (u16)read_sde_csr(sde, SD(HEAD));
1817			idle_check_done = 1;
1818			goto retry;
1819		}
1820	}
1821
1822	sde->last_status = status;
1823	if (progress)
1824		sdma_desc_avail(sde, sdma_descq_freecnt(sde));
1825}
1826
1827/*
1828 * sdma_engine_interrupt() - interrupt handler for engine
1829 * @sde: sdma engine
1830 * @status: sdma interrupt reason
1831 *
1832 * Status is a mask of the 3 possible interrupts for this engine.  It will
1833 * contain bits _only_ for this SDMA engine.  It will contain at least one
1834 * bit, it may contain more.
1835 */
1836void sdma_engine_interrupt(struct sdma_engine *sde, u64 status)
1837{
1838	trace_hfi1_sdma_engine_interrupt(sde, status);
1839	write_seqlock(&sde->head_lock);
1840	sdma_set_desc_cnt(sde, sdma_desct_intr);
1841	if (status & sde->idle_mask)
1842		sde->idle_int_cnt++;
1843	else if (status & sde->progress_mask)
1844		sde->progress_int_cnt++;
1845	else if (status & sde->int_mask)
1846		sde->sdma_int_cnt++;
1847	sdma_make_progress(sde, status);
1848	write_sequnlock(&sde->head_lock);
1849}
1850
1851/**
1852 * sdma_engine_error() - error handler for engine
1853 * @sde: sdma engine
1854 * @status: sdma interrupt reason
1855 */
1856void sdma_engine_error(struct sdma_engine *sde, u64 status)
1857{
1858	unsigned long flags;
1859
1860#ifdef CONFIG_SDMA_VERBOSITY
1861	dd_dev_err(sde->dd, "CONFIG SDMA(%u) error status 0x%llx state %s\n",
1862		   sde->this_idx,
1863		   (unsigned long long)status,
1864		   sdma_state_names[sde->state.current_state]);
1865#endif
1866	spin_lock_irqsave(&sde->tail_lock, flags);
1867	write_seqlock(&sde->head_lock);
1868	if (status & ALL_SDMA_ENG_HALT_ERRS)
1869		__sdma_process_event(sde, sdma_event_e60_hw_halted);
1870	if (status & ~SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK)) {
1871		dd_dev_err(sde->dd,
1872			   "SDMA (%u) engine error: 0x%llx state %s\n",
1873			   sde->this_idx,
1874			   (unsigned long long)status,
1875			   sdma_state_names[sde->state.current_state]);
1876		dump_sdma_state(sde);
1877	}
1878	write_sequnlock(&sde->head_lock);
1879	spin_unlock_irqrestore(&sde->tail_lock, flags);
1880}
1881
1882static void sdma_sendctrl(struct sdma_engine *sde, unsigned op)
1883{
1884	u64 set_senddmactrl = 0;
1885	u64 clr_senddmactrl = 0;
1886	unsigned long flags;
1887
1888#ifdef CONFIG_SDMA_VERBOSITY
1889	dd_dev_err(sde->dd, "CONFIG SDMA(%u) senddmactrl E=%d I=%d H=%d C=%d\n",
1890		   sde->this_idx,
1891		   (op & SDMA_SENDCTRL_OP_ENABLE) ? 1 : 0,
1892		   (op & SDMA_SENDCTRL_OP_INTENABLE) ? 1 : 0,
1893		   (op & SDMA_SENDCTRL_OP_HALT) ? 1 : 0,
1894		   (op & SDMA_SENDCTRL_OP_CLEANUP) ? 1 : 0);
1895#endif
1896
1897	if (op & SDMA_SENDCTRL_OP_ENABLE)
1898		set_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK);
1899	else
1900		clr_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK);
1901
1902	if (op & SDMA_SENDCTRL_OP_INTENABLE)
1903		set_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK);
1904	else
1905		clr_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK);
1906
1907	if (op & SDMA_SENDCTRL_OP_HALT)
1908		set_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK);
1909	else
1910		clr_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK);
1911
1912	spin_lock_irqsave(&sde->senddmactrl_lock, flags);
1913
1914	sde->p_senddmactrl |= set_senddmactrl;
1915	sde->p_senddmactrl &= ~clr_senddmactrl;
1916
1917	if (op & SDMA_SENDCTRL_OP_CLEANUP)
1918		write_sde_csr(sde, SD(CTRL),
1919			      sde->p_senddmactrl |
1920			      SD(CTRL_SDMA_CLEANUP_SMASK));
1921	else
1922		write_sde_csr(sde, SD(CTRL), sde->p_senddmactrl);
1923
1924	spin_unlock_irqrestore(&sde->senddmactrl_lock, flags);
1925
1926#ifdef CONFIG_SDMA_VERBOSITY
1927	sdma_dumpstate(sde);
1928#endif
1929}
1930
1931static void sdma_setlengen(struct sdma_engine *sde)
1932{
1933#ifdef CONFIG_SDMA_VERBOSITY
1934	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
1935		   sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
1936#endif
1937
1938	/*
1939	 * Set SendDmaLenGen and clear-then-set the MSB of the generation
1940	 * count to enable generation checking and load the internal
1941	 * generation counter.
1942	 */
1943	write_sde_csr(sde, SD(LEN_GEN),
1944		      (sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT));
1945	write_sde_csr(sde, SD(LEN_GEN),
1946		      ((sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)) |
1947		      (4ULL << SD(LEN_GEN_GENERATION_SHIFT)));
1948}
1949
1950static inline void sdma_update_tail(struct sdma_engine *sde, u16 tail)
1951{
1952	/* Commit writes to memory and advance the tail on the chip */
1953	smp_wmb(); /* see get_txhead() */
1954	writeq(tail, sde->tail_csr);
1955}
1956
1957/*
1958 * This is called when changing to state s10_hw_start_up_halt_wait as
1959 * a result of send buffer errors or send DMA descriptor errors.
1960 */
1961static void sdma_hw_start_up(struct sdma_engine *sde)
1962{
1963	u64 reg;
1964
1965#ifdef CONFIG_SDMA_VERBOSITY
1966	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
1967		   sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
1968#endif
1969
1970	sdma_setlengen(sde);
1971	sdma_update_tail(sde, 0); /* Set SendDmaTail */
1972	*sde->head_dma = 0;
1973
1974	reg = SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_MASK) <<
1975	      SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SHIFT);
1976	write_sde_csr(sde, SD(ENG_ERR_CLEAR), reg);
1977}
1978
1979/*
1980 * set_sdma_integrity
1981 *
1982 * Set the SEND_DMA_CHECK_ENABLE register for send DMA engine 'sde'.
1983 */
1984static void set_sdma_integrity(struct sdma_engine *sde)
1985{
1986	struct hfi1_devdata *dd = sde->dd;
1987
1988	write_sde_csr(sde, SD(CHECK_ENABLE),
1989		      hfi1_pkt_base_sdma_integrity(dd));
1990}
1991
1992static void init_sdma_regs(
1993	struct sdma_engine *sde,
1994	u32 credits,
1995	uint idle_cnt)
1996{
1997	u8 opval, opmask;
1998#ifdef CONFIG_SDMA_VERBOSITY
1999	struct hfi1_devdata *dd = sde->dd;
2000
2001	dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n",
2002		   sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
2003#endif
2004
2005	write_sde_csr(sde, SD(BASE_ADDR), sde->descq_phys);
2006	sdma_setlengen(sde);
2007	sdma_update_tail(sde, 0); /* Set SendDmaTail */
2008	write_sde_csr(sde, SD(RELOAD_CNT), idle_cnt);
2009	write_sde_csr(sde, SD(DESC_CNT), 0);
2010	write_sde_csr(sde, SD(HEAD_ADDR), sde->head_phys);
2011	write_sde_csr(sde, SD(MEMORY),
2012		      ((u64)credits << SD(MEMORY_SDMA_MEMORY_CNT_SHIFT)) |
2013		      ((u64)(credits * sde->this_idx) <<
2014		       SD(MEMORY_SDMA_MEMORY_INDEX_SHIFT)));
2015	write_sde_csr(sde, SD(ENG_ERR_MASK), ~0ull);
2016	set_sdma_integrity(sde);
2017	opmask = OPCODE_CHECK_MASK_DISABLED;
2018	opval = OPCODE_CHECK_VAL_DISABLED;
2019	write_sde_csr(sde, SD(CHECK_OPCODE),
2020		      (opmask << SEND_CTXT_CHECK_OPCODE_MASK_SHIFT) |
2021		      (opval << SEND_CTXT_CHECK_OPCODE_VALUE_SHIFT));
2022}
2023
2024#ifdef CONFIG_SDMA_VERBOSITY
2025
2026#define sdma_dumpstate_helper0(reg) do { \
2027		csr = read_csr(sde->dd, reg); \
2028		dd_dev_err(sde->dd, "%36s     0x%016llx\n", #reg, csr); \
2029	} while (0)
2030
2031#define sdma_dumpstate_helper(reg) do { \
2032		csr = read_sde_csr(sde, reg); \
2033		dd_dev_err(sde->dd, "%36s[%02u] 0x%016llx\n", \
2034			#reg, sde->this_idx, csr); \
2035	} while (0)
2036
2037#define sdma_dumpstate_helper2(reg) do { \
2038		csr = read_csr(sde->dd, reg + (8 * i)); \
2039		dd_dev_err(sde->dd, "%33s_%02u     0x%016llx\n", \
2040				#reg, i, csr); \
2041	} while (0)
2042
2043void sdma_dumpstate(struct sdma_engine *sde)
2044{
2045	u64 csr;
2046	unsigned i;
2047
2048	sdma_dumpstate_helper(SD(CTRL));
2049	sdma_dumpstate_helper(SD(STATUS));
2050	sdma_dumpstate_helper0(SD(ERR_STATUS));
2051	sdma_dumpstate_helper0(SD(ERR_MASK));
2052	sdma_dumpstate_helper(SD(ENG_ERR_STATUS));
2053	sdma_dumpstate_helper(SD(ENG_ERR_MASK));
2054
2055	for (i = 0; i < CCE_NUM_INT_CSRS; ++i) {
2056		sdma_dumpstate_helper2(CCE_INT_STATUS);
2057		sdma_dumpstate_helper2(CCE_INT_MASK);
2058		sdma_dumpstate_helper2(CCE_INT_BLOCKED);
2059	}
2060
2061	sdma_dumpstate_helper(SD(TAIL));
2062	sdma_dumpstate_helper(SD(HEAD));
2063	sdma_dumpstate_helper(SD(PRIORITY_THLD));
2064	sdma_dumpstate_helper(SD(IDLE_CNT));
2065	sdma_dumpstate_helper(SD(RELOAD_CNT));
2066	sdma_dumpstate_helper(SD(DESC_CNT));
2067	sdma_dumpstate_helper(SD(DESC_FETCHED_CNT));
2068	sdma_dumpstate_helper(SD(MEMORY));
2069	sdma_dumpstate_helper0(SD(ENGINES));
2070	sdma_dumpstate_helper0(SD(MEM_SIZE));
2071	/* sdma_dumpstate_helper(SEND_EGRESS_SEND_DMA_STATUS);  */
2072	sdma_dumpstate_helper(SD(BASE_ADDR));
2073	sdma_dumpstate_helper(SD(LEN_GEN));
2074	sdma_dumpstate_helper(SD(HEAD_ADDR));
2075	sdma_dumpstate_helper(SD(CHECK_ENABLE));
2076	sdma_dumpstate_helper(SD(CHECK_VL));
2077	sdma_dumpstate_helper(SD(CHECK_JOB_KEY));
2078	sdma_dumpstate_helper(SD(CHECK_PARTITION_KEY));
2079	sdma_dumpstate_helper(SD(CHECK_SLID));
2080	sdma_dumpstate_helper(SD(CHECK_OPCODE));
2081}
2082#endif
2083
2084static void dump_sdma_state(struct sdma_engine *sde)
2085{
2086	struct hw_sdma_desc *descqp;
2087	u64 desc[2];
2088	u64 addr;
2089	u8 gen;
2090	u16 len;
2091	u16 head, tail, cnt;
2092
2093	head = sde->descq_head & sde->sdma_mask;
2094	tail = sde->descq_tail & sde->sdma_mask;
2095	cnt = sdma_descq_freecnt(sde);
2096
2097	dd_dev_err(sde->dd,
2098		   "SDMA (%u) descq_head: %u descq_tail: %u freecnt: %u FLE %d\n",
2099		   sde->this_idx, head, tail, cnt,
2100		   !list_empty(&sde->flushlist));
2101
2102	/* print info for each entry in the descriptor queue */
2103	while (head != tail) {
2104		char flags[6] = { 'x', 'x', 'x', 'x', 0 };
2105
2106		descqp = &sde->descq[head];
2107		desc[0] = le64_to_cpu(descqp->qw[0]);
2108		desc[1] = le64_to_cpu(descqp->qw[1]);
2109		flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-';
2110		flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ?
2111				'H' : '-';
2112		flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-';
2113		flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-';
2114		addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT)
2115			& SDMA_DESC0_PHY_ADDR_MASK;
2116		gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT)
2117			& SDMA_DESC1_GENERATION_MASK;
2118		len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT)
2119			& SDMA_DESC0_BYTE_COUNT_MASK;
2120		dd_dev_err(sde->dd,
2121			   "SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n",
2122			   head, flags, addr, gen, len);
2123		dd_dev_err(sde->dd,
2124			   "\tdesc0:0x%016llx desc1 0x%016llx\n",
2125			   desc[0], desc[1]);
2126		if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG)
2127			dd_dev_err(sde->dd,
2128				   "\taidx: %u amode: %u alen: %u\n",
2129				   (u8)((desc[1] &
2130					 SDMA_DESC1_HEADER_INDEX_SMASK) >>
2131					SDMA_DESC1_HEADER_INDEX_SHIFT),
2132				   (u8)((desc[1] &
2133					 SDMA_DESC1_HEADER_MODE_SMASK) >>
2134					SDMA_DESC1_HEADER_MODE_SHIFT),
2135				   (u8)((desc[1] &
2136					 SDMA_DESC1_HEADER_DWS_SMASK) >>
2137					SDMA_DESC1_HEADER_DWS_SHIFT));
2138		head++;
2139		head &= sde->sdma_mask;
2140	}
2141}
2142
2143#define SDE_FMT \
2144	"SDE %u CPU %d STE %s C 0x%llx S 0x%016llx E 0x%llx T(HW) 0x%llx T(SW) 0x%x H(HW) 0x%llx H(SW) 0x%x H(D) 0x%llx DM 0x%llx GL 0x%llx R 0x%llx LIS 0x%llx AHGI 0x%llx TXT %u TXH %u DT %u DH %u FLNE %d DQF %u SLC 0x%llx\n"
2145/**
2146 * sdma_seqfile_dump_sde() - debugfs dump of sde
2147 * @s: seq file
2148 * @sde: send dma engine to dump
2149 *
2150 * This routine dumps the sde to the indicated seq file.
2151 */
2152void sdma_seqfile_dump_sde(struct seq_file *s, struct sdma_engine *sde)
2153{
2154	u16 head, tail;
2155	struct hw_sdma_desc *descqp;
2156	u64 desc[2];
2157	u64 addr;
2158	u8 gen;
2159	u16 len;
2160
2161	head = sde->descq_head & sde->sdma_mask;
2162	tail = READ_ONCE(sde->descq_tail) & sde->sdma_mask;
2163	seq_printf(s, SDE_FMT, sde->this_idx,
2164		   sde->cpu,
2165		   sdma_state_name(sde->state.current_state),
2166		   (unsigned long long)read_sde_csr(sde, SD(CTRL)),
2167		   (unsigned long long)read_sde_csr(sde, SD(STATUS)),
2168		   (unsigned long long)read_sde_csr(sde, SD(ENG_ERR_STATUS)),
2169		   (unsigned long long)read_sde_csr(sde, SD(TAIL)), tail,
2170		   (unsigned long long)read_sde_csr(sde, SD(HEAD)), head,
2171		   (unsigned long long)le64_to_cpu(*sde->head_dma),
2172		   (unsigned long long)read_sde_csr(sde, SD(MEMORY)),
2173		   (unsigned long long)read_sde_csr(sde, SD(LEN_GEN)),
2174		   (unsigned long long)read_sde_csr(sde, SD(RELOAD_CNT)),
2175		   (unsigned long long)sde->last_status,
2176		   (unsigned long long)sde->ahg_bits,
2177		   sde->tx_tail,
2178		   sde->tx_head,
2179		   sde->descq_tail,
2180		   sde->descq_head,
2181		   !list_empty(&sde->flushlist),
2182		   sde->descq_full_count,
2183		   (unsigned long long)read_sde_csr(sde, SEND_DMA_CHECK_SLID));
2184
2185	/* print info for each entry in the descriptor queue */
2186	while (head != tail) {
2187		char flags[6] = { 'x', 'x', 'x', 'x', 0 };
2188
2189		descqp = &sde->descq[head];
2190		desc[0] = le64_to_cpu(descqp->qw[0]);
2191		desc[1] = le64_to_cpu(descqp->qw[1]);
2192		flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-';
2193		flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ?
2194				'H' : '-';
2195		flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-';
2196		flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-';
2197		addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT)
2198			& SDMA_DESC0_PHY_ADDR_MASK;
2199		gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT)
2200			& SDMA_DESC1_GENERATION_MASK;
2201		len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT)
2202			& SDMA_DESC0_BYTE_COUNT_MASK;
2203		seq_printf(s,
2204			   "\tdesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n",
2205			   head, flags, addr, gen, len);
2206		if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG)
2207			seq_printf(s, "\t\tahgidx: %u ahgmode: %u\n",
2208				   (u8)((desc[1] &
2209					 SDMA_DESC1_HEADER_INDEX_SMASK) >>
2210					SDMA_DESC1_HEADER_INDEX_SHIFT),
2211				   (u8)((desc[1] &
2212					 SDMA_DESC1_HEADER_MODE_SMASK) >>
2213					SDMA_DESC1_HEADER_MODE_SHIFT));
2214		head = (head + 1) & sde->sdma_mask;
2215	}
2216}
2217
2218/*
2219 * add the generation number into
2220 * the qw1 and return
2221 */
2222static inline u64 add_gen(struct sdma_engine *sde, u64 qw1)
2223{
2224	u8 generation = (sde->descq_tail >> sde->sdma_shift) & 3;
2225
2226	qw1 &= ~SDMA_DESC1_GENERATION_SMASK;
2227	qw1 |= ((u64)generation & SDMA_DESC1_GENERATION_MASK)
2228			<< SDMA_DESC1_GENERATION_SHIFT;
2229	return qw1;
2230}
2231
2232/*
2233 * This routine submits the indicated tx
2234 *
2235 * Space has already been guaranteed and
2236 * tail side of ring is locked.
2237 *
2238 * The hardware tail update is done
2239 * in the caller and that is facilitated
2240 * by returning the new tail.
2241 *
2242 * There is special case logic for ahg
2243 * to not add the generation number for
2244 * up to 2 descriptors that follow the
2245 * first descriptor.
2246 *
2247 */
2248static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx)
2249{
2250	int i;
2251	u16 tail;
2252	struct sdma_desc *descp = tx->descp;
2253	u8 skip = 0, mode = ahg_mode(tx);
2254
2255	tail = sde->descq_tail & sde->sdma_mask;
2256	sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]);
2257	sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1]));
2258	trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1],
2259				   tail, &sde->descq[tail]);
2260	tail = ++sde->descq_tail & sde->sdma_mask;
2261	descp++;
2262	if (mode > SDMA_AHG_APPLY_UPDATE1)
2263		skip = mode >> 1;
2264	for (i = 1; i < tx->num_desc; i++, descp++) {
2265		u64 qw1;
2266
2267		sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]);
2268		if (skip) {
2269			/* edits don't have generation */
2270			qw1 = descp->qw[1];
2271			skip--;
2272		} else {
2273			/* replace generation with real one for non-edits */
2274			qw1 = add_gen(sde, descp->qw[1]);
2275		}
2276		sde->descq[tail].qw[1] = cpu_to_le64(qw1);
2277		trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1,
2278					   tail, &sde->descq[tail]);
2279		tail = ++sde->descq_tail & sde->sdma_mask;
2280	}
2281	tx->next_descq_idx = tail;
2282#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
2283	tx->sn = sde->tail_sn++;
2284	trace_hfi1_sdma_in_sn(sde, tx->sn);
2285	WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]);
2286#endif
2287	sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx;
2288	sde->desc_avail -= tx->num_desc;
2289	return tail;
2290}
2291
2292/*
2293 * Check for progress
2294 */
2295static int sdma_check_progress(
2296	struct sdma_engine *sde,
2297	struct iowait_work *wait,
2298	struct sdma_txreq *tx,
2299	bool pkts_sent)
2300{
2301	int ret;
2302
2303	sde->desc_avail = sdma_descq_freecnt(sde);
2304	if (tx->num_desc <= sde->desc_avail)
2305		return -EAGAIN;
2306	/* pulse the head_lock */
2307	if (wait && iowait_ioww_to_iow(wait)->sleep) {
2308		unsigned seq;
2309
2310		seq = raw_seqcount_begin(
2311			(const seqcount_t *)&sde->head_lock.seqcount);
2312		ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent);
2313		if (ret == -EAGAIN)
2314			sde->desc_avail = sdma_descq_freecnt(sde);
2315	} else {
2316		ret = -EBUSY;
2317	}
2318	return ret;
2319}
2320
2321/**
2322 * sdma_send_txreq() - submit a tx req to ring
2323 * @sde: sdma engine to use
2324 * @wait: SE wait structure to use when full (may be NULL)
2325 * @tx: sdma_txreq to submit
2326 * @pkts_sent: has any packet been sent yet?
2327 *
2328 * The call submits the tx into the ring.  If a iowait structure is non-NULL
2329 * the packet will be queued to the list in wait.
2330 *
2331 * Return:
2332 * 0 - Success, -EINVAL - sdma_txreq incomplete, -EBUSY - no space in
2333 * ring (wait == NULL)
2334 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
2335 */
2336int sdma_send_txreq(struct sdma_engine *sde,
2337		    struct iowait_work *wait,
2338		    struct sdma_txreq *tx,
2339		    bool pkts_sent)
2340{
2341	int ret = 0;
2342	u16 tail;
2343	unsigned long flags;
2344
2345	/* user should have supplied entire packet */
2346	if (unlikely(tx->tlen))
2347		return -EINVAL;
2348	tx->wait = iowait_ioww_to_iow(wait);
2349	spin_lock_irqsave(&sde->tail_lock, flags);
2350retry:
2351	if (unlikely(!__sdma_running(sde)))
2352		goto unlock_noconn;
2353	if (unlikely(tx->num_desc > sde->desc_avail))
2354		goto nodesc;
2355	tail = submit_tx(sde, tx);
2356	if (wait)
2357		iowait_sdma_inc(iowait_ioww_to_iow(wait));
2358	sdma_update_tail(sde, tail);
2359unlock:
2360	spin_unlock_irqrestore(&sde->tail_lock, flags);
2361	return ret;
2362unlock_noconn:
2363	if (wait)
2364		iowait_sdma_inc(iowait_ioww_to_iow(wait));
2365	tx->next_descq_idx = 0;
2366#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
2367	tx->sn = sde->tail_sn++;
2368	trace_hfi1_sdma_in_sn(sde, tx->sn);
2369#endif
2370	spin_lock(&sde->flushlist_lock);
2371	list_add_tail(&tx->list, &sde->flushlist);
2372	spin_unlock(&sde->flushlist_lock);
2373	iowait_inc_wait_count(wait, tx->num_desc);
2374	queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker);
2375	ret = -ECOMM;
2376	goto unlock;
2377nodesc:
2378	ret = sdma_check_progress(sde, wait, tx, pkts_sent);
2379	if (ret == -EAGAIN) {
2380		ret = 0;
2381		goto retry;
2382	}
2383	sde->descq_full_count++;
2384	goto unlock;
2385}
2386
2387/**
2388 * sdma_send_txlist() - submit a list of tx req to ring
2389 * @sde: sdma engine to use
2390 * @wait: SE wait structure to use when full (may be NULL)
2391 * @tx_list: list of sdma_txreqs to submit
2392 * @count_out: pointer to a u16 which, after return will contain the total number of
2393 *             sdma_txreqs removed from the tx_list. This will include sdma_txreqs
2394 *             whose SDMA descriptors are submitted to the ring and the sdma_txreqs
2395 *             which are added to SDMA engine flush list if the SDMA engine state is
2396 *             not running.
2397 *
2398 * The call submits the list into the ring.
2399 *
2400 * If the iowait structure is non-NULL and not equal to the iowait list
2401 * the unprocessed part of the list  will be appended to the list in wait.
2402 *
2403 * In all cases, the tx_list will be updated so the head of the tx_list is
2404 * the list of descriptors that have yet to be transmitted.
2405 *
2406 * The intent of this call is to provide a more efficient
2407 * way of submitting multiple packets to SDMA while holding the tail
2408 * side locking.
2409 *
2410 * Return:
2411 * 0 - Success,
2412 * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL)
2413 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
2414 */
2415int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait,
2416		     struct list_head *tx_list, u16 *count_out)
2417{
2418	struct sdma_txreq *tx, *tx_next;
2419	int ret = 0;
2420	unsigned long flags;
2421	u16 tail = INVALID_TAIL;
2422	u32 submit_count = 0, flush_count = 0, total_count;
2423
2424	spin_lock_irqsave(&sde->tail_lock, flags);
2425retry:
2426	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2427		tx->wait = iowait_ioww_to_iow(wait);
2428		if (unlikely(!__sdma_running(sde)))
2429			goto unlock_noconn;
2430		if (unlikely(tx->num_desc > sde->desc_avail))
2431			goto nodesc;
2432		if (unlikely(tx->tlen)) {
2433			ret = -EINVAL;
2434			goto update_tail;
2435		}
2436		list_del_init(&tx->list);
2437		tail = submit_tx(sde, tx);
2438		submit_count++;
2439		if (tail != INVALID_TAIL &&
2440		    (submit_count & SDMA_TAIL_UPDATE_THRESH) == 0) {
2441			sdma_update_tail(sde, tail);
2442			tail = INVALID_TAIL;
2443		}
2444	}
2445update_tail:
2446	total_count = submit_count + flush_count;
2447	if (wait) {
2448		iowait_sdma_add(iowait_ioww_to_iow(wait), total_count);
2449		iowait_starve_clear(submit_count > 0,
2450				    iowait_ioww_to_iow(wait));
2451	}
2452	if (tail != INVALID_TAIL)
2453		sdma_update_tail(sde, tail);
2454	spin_unlock_irqrestore(&sde->tail_lock, flags);
2455	*count_out = total_count;
2456	return ret;
2457unlock_noconn:
2458	spin_lock(&sde->flushlist_lock);
2459	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2460		tx->wait = iowait_ioww_to_iow(wait);
2461		list_del_init(&tx->list);
2462		tx->next_descq_idx = 0;
2463#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
2464		tx->sn = sde->tail_sn++;
2465		trace_hfi1_sdma_in_sn(sde, tx->sn);
2466#endif
2467		list_add_tail(&tx->list, &sde->flushlist);
2468		flush_count++;
2469		iowait_inc_wait_count(wait, tx->num_desc);
2470	}
2471	spin_unlock(&sde->flushlist_lock);
2472	queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker);
2473	ret = -ECOMM;
2474	goto update_tail;
2475nodesc:
2476	ret = sdma_check_progress(sde, wait, tx, submit_count > 0);
2477	if (ret == -EAGAIN) {
2478		ret = 0;
2479		goto retry;
2480	}
2481	sde->descq_full_count++;
2482	goto update_tail;
2483}
2484
2485static void sdma_process_event(struct sdma_engine *sde, enum sdma_events event)
2486{
2487	unsigned long flags;
2488
2489	spin_lock_irqsave(&sde->tail_lock, flags);
2490	write_seqlock(&sde->head_lock);
2491
2492	__sdma_process_event(sde, event);
2493
2494	if (sde->state.current_state == sdma_state_s99_running)
2495		sdma_desc_avail(sde, sdma_descq_freecnt(sde));
2496
2497	write_sequnlock(&sde->head_lock);
2498	spin_unlock_irqrestore(&sde->tail_lock, flags);
2499}
2500
2501static void __sdma_process_event(struct sdma_engine *sde,
2502				 enum sdma_events event)
2503{
2504	struct sdma_state *ss = &sde->state;
2505	int need_progress = 0;
2506
2507	/* CONFIG SDMA temporary */
2508#ifdef CONFIG_SDMA_VERBOSITY
2509	dd_dev_err(sde->dd, "CONFIG SDMA(%u) [%s] %s\n", sde->this_idx,
2510		   sdma_state_names[ss->current_state],
2511		   sdma_event_names[event]);
2512#endif
2513
2514	switch (ss->current_state) {
2515	case sdma_state_s00_hw_down:
2516		switch (event) {
2517		case sdma_event_e00_go_hw_down:
2518			break;
2519		case sdma_event_e30_go_running:
2520			/*
2521			 * If down, but running requested (usually result
2522			 * of link up, then we need to start up.
2523			 * This can happen when hw down is requested while
2524			 * bringing the link up with traffic active on
2525			 * 7220, e.g.
2526			 */
2527			ss->go_s99_running = 1;
2528			fallthrough;	/* and start dma engine */
2529		case sdma_event_e10_go_hw_start:
2530			/* This reference means the state machine is started */
2531			sdma_get(&sde->state);
2532			sdma_set_state(sde,
2533				       sdma_state_s10_hw_start_up_halt_wait);
2534			break;
2535		case sdma_event_e15_hw_halt_done:
2536			break;
2537		case sdma_event_e25_hw_clean_up_done:
2538			break;
2539		case sdma_event_e40_sw_cleaned:
2540			sdma_sw_tear_down(sde);
2541			break;
2542		case sdma_event_e50_hw_cleaned:
2543			break;
2544		case sdma_event_e60_hw_halted:
2545			break;
2546		case sdma_event_e70_go_idle:
2547			break;
2548		case sdma_event_e80_hw_freeze:
2549			break;
2550		case sdma_event_e81_hw_frozen:
2551			break;
2552		case sdma_event_e82_hw_unfreeze:
2553			break;
2554		case sdma_event_e85_link_down:
2555			break;
2556		case sdma_event_e90_sw_halted:
2557			break;
2558		}
2559		break;
2560
2561	case sdma_state_s10_hw_start_up_halt_wait:
2562		switch (event) {
2563		case sdma_event_e00_go_hw_down:
2564			sdma_set_state(sde, sdma_state_s00_hw_down);
2565			sdma_sw_tear_down(sde);
2566			break;
2567		case sdma_event_e10_go_hw_start:
2568			break;
2569		case sdma_event_e15_hw_halt_done:
2570			sdma_set_state(sde,
2571				       sdma_state_s15_hw_start_up_clean_wait);
2572			sdma_start_hw_clean_up(sde);
2573			break;
2574		case sdma_event_e25_hw_clean_up_done:
2575			break;
2576		case sdma_event_e30_go_running:
2577			ss->go_s99_running = 1;
2578			break;
2579		case sdma_event_e40_sw_cleaned:
2580			break;
2581		case sdma_event_e50_hw_cleaned:
2582			break;
2583		case sdma_event_e60_hw_halted:
2584			schedule_work(&sde->err_halt_worker);
2585			break;
2586		case sdma_event_e70_go_idle:
2587			ss->go_s99_running = 0;
2588			break;
2589		case sdma_event_e80_hw_freeze:
2590			break;
2591		case sdma_event_e81_hw_frozen:
2592			break;
2593		case sdma_event_e82_hw_unfreeze:
2594			break;
2595		case sdma_event_e85_link_down:
2596			break;
2597		case sdma_event_e90_sw_halted:
2598			break;
2599		}
2600		break;
2601
2602	case sdma_state_s15_hw_start_up_clean_wait:
2603		switch (event) {
2604		case sdma_event_e00_go_hw_down:
2605			sdma_set_state(sde, sdma_state_s00_hw_down);
2606			sdma_sw_tear_down(sde);
2607			break;
2608		case sdma_event_e10_go_hw_start:
2609			break;
2610		case sdma_event_e15_hw_halt_done:
2611			break;
2612		case sdma_event_e25_hw_clean_up_done:
2613			sdma_hw_start_up(sde);
2614			sdma_set_state(sde, ss->go_s99_running ?
2615				       sdma_state_s99_running :
2616				       sdma_state_s20_idle);
2617			break;
2618		case sdma_event_e30_go_running:
2619			ss->go_s99_running = 1;
2620			break;
2621		case sdma_event_e40_sw_cleaned:
2622			break;
2623		case sdma_event_e50_hw_cleaned:
2624			break;
2625		case sdma_event_e60_hw_halted:
2626			break;
2627		case sdma_event_e70_go_idle:
2628			ss->go_s99_running = 0;
2629			break;
2630		case sdma_event_e80_hw_freeze:
2631			break;
2632		case sdma_event_e81_hw_frozen:
2633			break;
2634		case sdma_event_e82_hw_unfreeze:
2635			break;
2636		case sdma_event_e85_link_down:
2637			break;
2638		case sdma_event_e90_sw_halted:
2639			break;
2640		}
2641		break;
2642
2643	case sdma_state_s20_idle:
2644		switch (event) {
2645		case sdma_event_e00_go_hw_down:
2646			sdma_set_state(sde, sdma_state_s00_hw_down);
2647			sdma_sw_tear_down(sde);
2648			break;
2649		case sdma_event_e10_go_hw_start:
2650			break;
2651		case sdma_event_e15_hw_halt_done:
2652			break;
2653		case sdma_event_e25_hw_clean_up_done:
2654			break;
2655		case sdma_event_e30_go_running:
2656			sdma_set_state(sde, sdma_state_s99_running);
2657			ss->go_s99_running = 1;
2658			break;
2659		case sdma_event_e40_sw_cleaned:
2660			break;
2661		case sdma_event_e50_hw_cleaned:
2662			break;
2663		case sdma_event_e60_hw_halted:
2664			sdma_set_state(sde, sdma_state_s50_hw_halt_wait);
2665			schedule_work(&sde->err_halt_worker);
2666			break;
2667		case sdma_event_e70_go_idle:
2668			break;
2669		case sdma_event_e85_link_down:
2670		case sdma_event_e80_hw_freeze:
2671			sdma_set_state(sde, sdma_state_s80_hw_freeze);
2672			atomic_dec(&sde->dd->sdma_unfreeze_count);
2673			wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
2674			break;
2675		case sdma_event_e81_hw_frozen:
2676			break;
2677		case sdma_event_e82_hw_unfreeze:
2678			break;
2679		case sdma_event_e90_sw_halted:
2680			break;
2681		}
2682		break;
2683
2684	case sdma_state_s30_sw_clean_up_wait:
2685		switch (event) {
2686		case sdma_event_e00_go_hw_down:
2687			sdma_set_state(sde, sdma_state_s00_hw_down);
2688			break;
2689		case sdma_event_e10_go_hw_start:
2690			break;
2691		case sdma_event_e15_hw_halt_done:
2692			break;
2693		case sdma_event_e25_hw_clean_up_done:
2694			break;
2695		case sdma_event_e30_go_running:
2696			ss->go_s99_running = 1;
2697			break;
2698		case sdma_event_e40_sw_cleaned:
2699			sdma_set_state(sde, sdma_state_s40_hw_clean_up_wait);
2700			sdma_start_hw_clean_up(sde);
2701			break;
2702		case sdma_event_e50_hw_cleaned:
2703			break;
2704		case sdma_event_e60_hw_halted:
2705			break;
2706		case sdma_event_e70_go_idle:
2707			ss->go_s99_running = 0;
2708			break;
2709		case sdma_event_e80_hw_freeze:
2710			break;
2711		case sdma_event_e81_hw_frozen:
2712			break;
2713		case sdma_event_e82_hw_unfreeze:
2714			break;
2715		case sdma_event_e85_link_down:
2716			ss->go_s99_running = 0;
2717			break;
2718		case sdma_event_e90_sw_halted:
2719			break;
2720		}
2721		break;
2722
2723	case sdma_state_s40_hw_clean_up_wait:
2724		switch (event) {
2725		case sdma_event_e00_go_hw_down:
2726			sdma_set_state(sde, sdma_state_s00_hw_down);
2727			tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2728			break;
2729		case sdma_event_e10_go_hw_start:
2730			break;
2731		case sdma_event_e15_hw_halt_done:
2732			break;
2733		case sdma_event_e25_hw_clean_up_done:
2734			sdma_hw_start_up(sde);
2735			sdma_set_state(sde, ss->go_s99_running ?
2736				       sdma_state_s99_running :
2737				       sdma_state_s20_idle);
2738			break;
2739		case sdma_event_e30_go_running:
2740			ss->go_s99_running = 1;
2741			break;
2742		case sdma_event_e40_sw_cleaned:
2743			break;
2744		case sdma_event_e50_hw_cleaned:
2745			break;
2746		case sdma_event_e60_hw_halted:
2747			break;
2748		case sdma_event_e70_go_idle:
2749			ss->go_s99_running = 0;
2750			break;
2751		case sdma_event_e80_hw_freeze:
2752			break;
2753		case sdma_event_e81_hw_frozen:
2754			break;
2755		case sdma_event_e82_hw_unfreeze:
2756			break;
2757		case sdma_event_e85_link_down:
2758			ss->go_s99_running = 0;
2759			break;
2760		case sdma_event_e90_sw_halted:
2761			break;
2762		}
2763		break;
2764
2765	case sdma_state_s50_hw_halt_wait:
2766		switch (event) {
2767		case sdma_event_e00_go_hw_down:
2768			sdma_set_state(sde, sdma_state_s00_hw_down);
2769			tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2770			break;
2771		case sdma_event_e10_go_hw_start:
2772			break;
2773		case sdma_event_e15_hw_halt_done:
2774			sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait);
2775			tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2776			break;
2777		case sdma_event_e25_hw_clean_up_done:
2778			break;
2779		case sdma_event_e30_go_running:
2780			ss->go_s99_running = 1;
2781			break;
2782		case sdma_event_e40_sw_cleaned:
2783			break;
2784		case sdma_event_e50_hw_cleaned:
2785			break;
2786		case sdma_event_e60_hw_halted:
2787			schedule_work(&sde->err_halt_worker);
2788			break;
2789		case sdma_event_e70_go_idle:
2790			ss->go_s99_running = 0;
2791			break;
2792		case sdma_event_e80_hw_freeze:
2793			break;
2794		case sdma_event_e81_hw_frozen:
2795			break;
2796		case sdma_event_e82_hw_unfreeze:
2797			break;
2798		case sdma_event_e85_link_down:
2799			ss->go_s99_running = 0;
2800			break;
2801		case sdma_event_e90_sw_halted:
2802			break;
2803		}
2804		break;
2805
2806	case sdma_state_s60_idle_halt_wait:
2807		switch (event) {
2808		case sdma_event_e00_go_hw_down:
2809			sdma_set_state(sde, sdma_state_s00_hw_down);
2810			tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2811			break;
2812		case sdma_event_e10_go_hw_start:
2813			break;
2814		case sdma_event_e15_hw_halt_done:
2815			sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait);
2816			tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2817			break;
2818		case sdma_event_e25_hw_clean_up_done:
2819			break;
2820		case sdma_event_e30_go_running:
2821			ss->go_s99_running = 1;
2822			break;
2823		case sdma_event_e40_sw_cleaned:
2824			break;
2825		case sdma_event_e50_hw_cleaned:
2826			break;
2827		case sdma_event_e60_hw_halted:
2828			schedule_work(&sde->err_halt_worker);
2829			break;
2830		case sdma_event_e70_go_idle:
2831			ss->go_s99_running = 0;
2832			break;
2833		case sdma_event_e80_hw_freeze:
2834			break;
2835		case sdma_event_e81_hw_frozen:
2836			break;
2837		case sdma_event_e82_hw_unfreeze:
2838			break;
2839		case sdma_event_e85_link_down:
2840			break;
2841		case sdma_event_e90_sw_halted:
2842			break;
2843		}
2844		break;
2845
2846	case sdma_state_s80_hw_freeze:
2847		switch (event) {
2848		case sdma_event_e00_go_hw_down:
2849			sdma_set_state(sde, sdma_state_s00_hw_down);
2850			tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2851			break;
2852		case sdma_event_e10_go_hw_start:
2853			break;
2854		case sdma_event_e15_hw_halt_done:
2855			break;
2856		case sdma_event_e25_hw_clean_up_done:
2857			break;
2858		case sdma_event_e30_go_running:
2859			ss->go_s99_running = 1;
2860			break;
2861		case sdma_event_e40_sw_cleaned:
2862			break;
2863		case sdma_event_e50_hw_cleaned:
2864			break;
2865		case sdma_event_e60_hw_halted:
2866			break;
2867		case sdma_event_e70_go_idle:
2868			ss->go_s99_running = 0;
2869			break;
2870		case sdma_event_e80_hw_freeze:
2871			break;
2872		case sdma_event_e81_hw_frozen:
2873			sdma_set_state(sde, sdma_state_s82_freeze_sw_clean);
2874			tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2875			break;
2876		case sdma_event_e82_hw_unfreeze:
2877			break;
2878		case sdma_event_e85_link_down:
2879			break;
2880		case sdma_event_e90_sw_halted:
2881			break;
2882		}
2883		break;
2884
2885	case sdma_state_s82_freeze_sw_clean:
2886		switch (event) {
2887		case sdma_event_e00_go_hw_down:
2888			sdma_set_state(sde, sdma_state_s00_hw_down);
2889			tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2890			break;
2891		case sdma_event_e10_go_hw_start:
2892			break;
2893		case sdma_event_e15_hw_halt_done:
2894			break;
2895		case sdma_event_e25_hw_clean_up_done:
2896			break;
2897		case sdma_event_e30_go_running:
2898			ss->go_s99_running = 1;
2899			break;
2900		case sdma_event_e40_sw_cleaned:
2901			/* notify caller this engine is done cleaning */
2902			atomic_dec(&sde->dd->sdma_unfreeze_count);
2903			wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
2904			break;
2905		case sdma_event_e50_hw_cleaned:
2906			break;
2907		case sdma_event_e60_hw_halted:
2908			break;
2909		case sdma_event_e70_go_idle:
2910			ss->go_s99_running = 0;
2911			break;
2912		case sdma_event_e80_hw_freeze:
2913			break;
2914		case sdma_event_e81_hw_frozen:
2915			break;
2916		case sdma_event_e82_hw_unfreeze:
2917			sdma_hw_start_up(sde);
2918			sdma_set_state(sde, ss->go_s99_running ?
2919				       sdma_state_s99_running :
2920				       sdma_state_s20_idle);
2921			break;
2922		case sdma_event_e85_link_down:
2923			break;
2924		case sdma_event_e90_sw_halted:
2925			break;
2926		}
2927		break;
2928
2929	case sdma_state_s99_running:
2930		switch (event) {
2931		case sdma_event_e00_go_hw_down:
2932			sdma_set_state(sde, sdma_state_s00_hw_down);
2933			tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2934			break;
2935		case sdma_event_e10_go_hw_start:
2936			break;
2937		case sdma_event_e15_hw_halt_done:
2938			break;
2939		case sdma_event_e25_hw_clean_up_done:
2940			break;
2941		case sdma_event_e30_go_running:
2942			break;
2943		case sdma_event_e40_sw_cleaned:
2944			break;
2945		case sdma_event_e50_hw_cleaned:
2946			break;
2947		case sdma_event_e60_hw_halted:
2948			need_progress = 1;
2949			sdma_err_progress_check_schedule(sde);
2950			fallthrough;
2951		case sdma_event_e90_sw_halted:
2952			/*
2953			* SW initiated halt does not perform engines
2954			* progress check
2955			*/
2956			sdma_set_state(sde, sdma_state_s50_hw_halt_wait);
2957			schedule_work(&sde->err_halt_worker);
2958			break;
2959		case sdma_event_e70_go_idle:
2960			sdma_set_state(sde, sdma_state_s60_idle_halt_wait);
2961			break;
2962		case sdma_event_e85_link_down:
2963			ss->go_s99_running = 0;
2964			fallthrough;
2965		case sdma_event_e80_hw_freeze:
2966			sdma_set_state(sde, sdma_state_s80_hw_freeze);
2967			atomic_dec(&sde->dd->sdma_unfreeze_count);
2968			wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
2969			break;
2970		case sdma_event_e81_hw_frozen:
2971			break;
2972		case sdma_event_e82_hw_unfreeze:
2973			break;
2974		}
2975		break;
2976	}
2977
2978	ss->last_event = event;
2979	if (need_progress)
2980		sdma_make_progress(sde, 0);
2981}
2982
2983/*
2984 * _extend_sdma_tx_descs() - helper to extend txreq
2985 *
2986 * This is called once the initial nominal allocation
2987 * of descriptors in the sdma_txreq is exhausted.
2988 *
2989 * The code will bump the allocation up to the max
2990 * of MAX_DESC (64) descriptors. There doesn't seem
2991 * much point in an interim step. The last descriptor
2992 * is reserved for coalesce buffer in order to support
2993 * cases where input packet has >MAX_DESC iovecs.
2994 *
2995 */
2996static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
2997{
2998	int i;
2999	struct sdma_desc *descp;
3000
3001	/* Handle last descriptor */
3002	if (unlikely((tx->num_desc == (MAX_DESC - 1)))) {
3003		/* if tlen is 0, it is for padding, release last descriptor */
3004		if (!tx->tlen) {
3005			tx->desc_limit = MAX_DESC;
3006		} else if (!tx->coalesce_buf) {
3007			/* allocate coalesce buffer with space for padding */
3008			tx->coalesce_buf = kmalloc(tx->tlen + sizeof(u32),
3009						   GFP_ATOMIC);
3010			if (!tx->coalesce_buf)
3011				goto enomem;
3012			tx->coalesce_idx = 0;
3013		}
3014		return 0;
3015	}
3016
3017	if (unlikely(tx->num_desc == MAX_DESC))
3018		goto enomem;
3019
3020	descp = kmalloc_array(MAX_DESC, sizeof(struct sdma_desc), GFP_ATOMIC);
3021	if (!descp)
3022		goto enomem;
3023	tx->descp = descp;
3024
3025	/* reserve last descriptor for coalescing */
3026	tx->desc_limit = MAX_DESC - 1;
3027	/* copy ones already built */
3028	for (i = 0; i < tx->num_desc; i++)
3029		tx->descp[i] = tx->descs[i];
3030	return 0;
3031enomem:
3032	__sdma_txclean(dd, tx);
3033	return -ENOMEM;
3034}
3035
3036/*
3037 * ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors
3038 *
3039 * This is called once the initial nominal allocation of descriptors
3040 * in the sdma_txreq is exhausted.
3041 *
3042 * This function calls _extend_sdma_tx_descs to extend or allocate
3043 * coalesce buffer. If there is a allocated coalesce buffer, it will
3044 * copy the input packet data into the coalesce buffer. It also adds
3045 * coalesce buffer descriptor once when whole packet is received.
3046 *
3047 * Return:
3048 * <0 - error
3049 * 0 - coalescing, don't populate descriptor
3050 * 1 - continue with populating descriptor
3051 */
3052int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
3053			   int type, void *kvaddr, struct page *page,
3054			   unsigned long offset, u16 len)
3055{
3056	int pad_len, rval;
3057	dma_addr_t addr;
3058
3059	rval = _extend_sdma_tx_descs(dd, tx);
3060	if (rval) {
3061		__sdma_txclean(dd, tx);
3062		return rval;
3063	}
3064
3065	/* If coalesce buffer is allocated, copy data into it */
3066	if (tx->coalesce_buf) {
3067		if (type == SDMA_MAP_NONE) {
3068			__sdma_txclean(dd, tx);
3069			return -EINVAL;
3070		}
3071
3072		if (type == SDMA_MAP_PAGE) {
3073			kvaddr = kmap_local_page(page);
3074			kvaddr += offset;
3075		} else if (WARN_ON(!kvaddr)) {
3076			__sdma_txclean(dd, tx);
3077			return -EINVAL;
3078		}
3079
3080		memcpy(tx->coalesce_buf + tx->coalesce_idx, kvaddr, len);
3081		tx->coalesce_idx += len;
3082		if (type == SDMA_MAP_PAGE)
3083			kunmap_local(kvaddr);
3084
3085		/* If there is more data, return */
3086		if (tx->tlen - tx->coalesce_idx)
3087			return 0;
3088
3089		/* Whole packet is received; add any padding */
3090		pad_len = tx->packet_len & (sizeof(u32) - 1);
3091		if (pad_len) {
3092			pad_len = sizeof(u32) - pad_len;
3093			memset(tx->coalesce_buf + tx->coalesce_idx, 0, pad_len);
3094			/* padding is taken care of for coalescing case */
3095			tx->packet_len += pad_len;
3096			tx->tlen += pad_len;
3097		}
3098
3099		/* dma map the coalesce buffer */
3100		addr = dma_map_single(&dd->pcidev->dev,
3101				      tx->coalesce_buf,
3102				      tx->tlen,
3103				      DMA_TO_DEVICE);
3104
3105		if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) {
3106			__sdma_txclean(dd, tx);
3107			return -ENOSPC;
3108		}
3109
3110		/* Add descriptor for coalesce buffer */
3111		tx->desc_limit = MAX_DESC;
3112		return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx,
3113					 addr, tx->tlen, NULL, NULL, NULL);
3114	}
3115
3116	return 1;
3117}
3118
3119/* Update sdes when the lmc changes */
3120void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid)
3121{
3122	struct sdma_engine *sde;
3123	int i;
3124	u64 sreg;
3125
3126	sreg = ((mask & SD(CHECK_SLID_MASK_MASK)) <<
3127		SD(CHECK_SLID_MASK_SHIFT)) |
3128		(((lid & mask) & SD(CHECK_SLID_VALUE_MASK)) <<
3129		SD(CHECK_SLID_VALUE_SHIFT));
3130
3131	for (i = 0; i < dd->num_sdma; i++) {
3132		hfi1_cdbg(LINKVERB, "SendDmaEngine[%d].SLID_CHECK = 0x%x",
3133			  i, (u32)sreg);
3134		sde = &dd->per_sdma[i];
3135		write_sde_csr(sde, SD(CHECK_SLID), sreg);
3136	}
3137}
3138
3139/* tx not dword sized - pad */
3140int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
3141{
3142	int rval = 0;
3143
3144	if ((unlikely(tx->num_desc == tx->desc_limit))) {
3145		rval = _extend_sdma_tx_descs(dd, tx);
3146		if (rval) {
3147			__sdma_txclean(dd, tx);
3148			return rval;
3149		}
3150	}
3151
3152	/* finish the one just added */
3153	make_tx_sdma_desc(
3154		tx,
3155		SDMA_MAP_NONE,
3156		dd->sdma_pad_phys,
3157		sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)),
3158		NULL, NULL, NULL);
3159	tx->num_desc++;
3160	_sdma_close_tx(dd, tx);
3161	return rval;
3162}
3163
3164/*
3165 * Add ahg to the sdma_txreq
3166 *
3167 * The logic will consume up to 3
3168 * descriptors at the beginning of
3169 * sdma_txreq.
3170 */
3171void _sdma_txreq_ahgadd(
3172	struct sdma_txreq *tx,
3173	u8 num_ahg,
3174	u8 ahg_entry,
3175	u32 *ahg,
3176	u8 ahg_hlen)
3177{
3178	u32 i, shift = 0, desc = 0;
3179	u8 mode;
3180
3181	WARN_ON_ONCE(num_ahg > 9 || (ahg_hlen & 3) || ahg_hlen == 4);
3182	/* compute mode */
3183	if (num_ahg == 1)
3184		mode = SDMA_AHG_APPLY_UPDATE1;
3185	else if (num_ahg <= 5)
3186		mode = SDMA_AHG_APPLY_UPDATE2;
3187	else
3188		mode = SDMA_AHG_APPLY_UPDATE3;
3189	tx->num_desc++;
3190	/* initialize to consumed descriptors to zero */
3191	switch (mode) {
3192	case SDMA_AHG_APPLY_UPDATE3:
3193		tx->num_desc++;
3194		tx->descs[2].qw[0] = 0;
3195		tx->descs[2].qw[1] = 0;
3196		fallthrough;
3197	case SDMA_AHG_APPLY_UPDATE2:
3198		tx->num_desc++;
3199		tx->descs[1].qw[0] = 0;
3200		tx->descs[1].qw[1] = 0;
3201		break;
3202	}
3203	ahg_hlen >>= 2;
3204	tx->descs[0].qw[1] |=
3205		(((u64)ahg_entry & SDMA_DESC1_HEADER_INDEX_MASK)
3206			<< SDMA_DESC1_HEADER_INDEX_SHIFT) |
3207		(((u64)ahg_hlen & SDMA_DESC1_HEADER_DWS_MASK)
3208			<< SDMA_DESC1_HEADER_DWS_SHIFT) |
3209		(((u64)mode & SDMA_DESC1_HEADER_MODE_MASK)
3210			<< SDMA_DESC1_HEADER_MODE_SHIFT) |
3211		(((u64)ahg[0] & SDMA_DESC1_HEADER_UPDATE1_MASK)
3212			<< SDMA_DESC1_HEADER_UPDATE1_SHIFT);
3213	for (i = 0; i < (num_ahg - 1); i++) {
3214		if (!shift && !(i & 2))
3215			desc++;
3216		tx->descs[desc].qw[!!(i & 2)] |=
3217			(((u64)ahg[i + 1])
3218				<< shift);
3219		shift = (shift + 32) & 63;
3220	}
3221}
3222
3223/**
3224 * sdma_ahg_alloc - allocate an AHG entry
3225 * @sde: engine to allocate from
3226 *
3227 * Return:
3228 * 0-31 when successful, -EOPNOTSUPP if AHG is not enabled,
3229 * -ENOSPC if an entry is not available
3230 */
3231int sdma_ahg_alloc(struct sdma_engine *sde)
3232{
3233	int nr;
3234	int oldbit;
3235
3236	if (!sde) {
3237		trace_hfi1_ahg_allocate(sde, -EINVAL);
3238		return -EINVAL;
3239	}
3240	while (1) {
3241		nr = ffz(READ_ONCE(sde->ahg_bits));
3242		if (nr > 31) {
3243			trace_hfi1_ahg_allocate(sde, -ENOSPC);
3244			return -ENOSPC;
3245		}
3246		oldbit = test_and_set_bit(nr, &sde->ahg_bits);
3247		if (!oldbit)
3248			break;
3249		cpu_relax();
3250	}
3251	trace_hfi1_ahg_allocate(sde, nr);
3252	return nr;
3253}
3254
3255/**
3256 * sdma_ahg_free - free an AHG entry
3257 * @sde: engine to return AHG entry
3258 * @ahg_index: index to free
3259 *
3260 * This routine frees the indicate AHG entry.
3261 */
3262void sdma_ahg_free(struct sdma_engine *sde, int ahg_index)
3263{
3264	if (!sde)
3265		return;
3266	trace_hfi1_ahg_deallocate(sde, ahg_index);
3267	if (ahg_index < 0 || ahg_index > 31)
3268		return;
3269	clear_bit(ahg_index, &sde->ahg_bits);
3270}
3271
3272/*
3273 * SPC freeze handling for SDMA engines.  Called when the driver knows
3274 * the SPC is going into a freeze but before the freeze is fully
3275 * settled.  Generally an error interrupt.
3276 *
3277 * This event will pull the engine out of running so no more entries can be
3278 * added to the engine's queue.
3279 */
3280void sdma_freeze_notify(struct hfi1_devdata *dd, int link_down)
3281{
3282	int i;
3283	enum sdma_events event = link_down ? sdma_event_e85_link_down :
3284					     sdma_event_e80_hw_freeze;
3285
3286	/* set up the wait but do not wait here */
3287	atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma);
3288
3289	/* tell all engines to stop running and wait */
3290	for (i = 0; i < dd->num_sdma; i++)
3291		sdma_process_event(&dd->per_sdma[i], event);
3292
3293	/* sdma_freeze() will wait for all engines to have stopped */
3294}
3295
3296/*
3297 * SPC freeze handling for SDMA engines.  Called when the driver knows
3298 * the SPC is fully frozen.
3299 */
3300void sdma_freeze(struct hfi1_devdata *dd)
3301{
3302	int i;
3303	int ret;
3304
3305	/*
3306	 * Make sure all engines have moved out of the running state before
3307	 * continuing.
3308	 */
3309	ret = wait_event_interruptible(dd->sdma_unfreeze_wq,
3310				       atomic_read(&dd->sdma_unfreeze_count) <=
3311				       0);
3312	/* interrupted or count is negative, then unloading - just exit */
3313	if (ret || atomic_read(&dd->sdma_unfreeze_count) < 0)
3314		return;
3315
3316	/* set up the count for the next wait */
3317	atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma);
3318
3319	/* tell all engines that the SPC is frozen, they can start cleaning */
3320	for (i = 0; i < dd->num_sdma; i++)
3321		sdma_process_event(&dd->per_sdma[i], sdma_event_e81_hw_frozen);
3322
3323	/*
3324	 * Wait for everyone to finish software clean before exiting.  The
3325	 * software clean will read engine CSRs, so must be completed before
3326	 * the next step, which will clear the engine CSRs.
3327	 */
3328	(void)wait_event_interruptible(dd->sdma_unfreeze_wq,
3329				atomic_read(&dd->sdma_unfreeze_count) <= 0);
3330	/* no need to check results - done no matter what */
3331}
3332
3333/*
3334 * SPC freeze handling for the SDMA engines.  Called after the SPC is unfrozen.
3335 *
3336 * The SPC freeze acts like a SDMA halt and a hardware clean combined.  All
3337 * that is left is a software clean.  We could do it after the SPC is fully
3338 * frozen, but then we'd have to add another state to wait for the unfreeze.
3339 * Instead, just defer the software clean until the unfreeze step.
3340 */
3341void sdma_unfreeze(struct hfi1_devdata *dd)
3342{
3343	int i;
3344
3345	/* tell all engines start freeze clean up */
3346	for (i = 0; i < dd->num_sdma; i++)
3347		sdma_process_event(&dd->per_sdma[i],
3348				   sdma_event_e82_hw_unfreeze);
3349}
3350
3351/**
3352 * _sdma_engine_progress_schedule() - schedule progress on engine
3353 * @sde: sdma_engine to schedule progress
3354 *
3355 */
3356void _sdma_engine_progress_schedule(
3357	struct sdma_engine *sde)
3358{
3359	trace_hfi1_sdma_engine_progress(sde, sde->progress_mask);
3360	/* assume we have selected a good cpu */
3361	write_csr(sde->dd,
3362		  CCE_INT_FORCE + (8 * (IS_SDMA_START / 64)),
3363		  sde->progress_mask);
3364}