ntb_hw_switchtec.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Microsemi Switchtec(tm) PCIe Management Driver
   4 * Copyright (c) 2017, Microsemi Corporation
   5 */
   6
   7#include <linux/interrupt.h>
   8#include <linux/io-64-nonatomic-lo-hi.h>
   9#include <linux/delay.h>
  10#include <linux/kthread.h>
  11#include <linux/module.h>
  12#include <linux/ntb.h>
  13#include <linux/pci.h>
  14#include <linux/switchtec.h>
  15
  16MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver");
  17MODULE_VERSION("0.1");
  18MODULE_LICENSE("GPL");
  19MODULE_AUTHOR("Microsemi Corporation");
  20
  21static ulong max_mw_size = SZ_2M;
  22module_param(max_mw_size, ulong, 0644);
  23MODULE_PARM_DESC(max_mw_size,
  24	"Max memory window size reported to the upper layer");
  25
  26static bool use_lut_mws;
  27module_param(use_lut_mws, bool, 0644);
  28MODULE_PARM_DESC(use_lut_mws,
  29		 "Enable the use of the LUT based memory windows");
  30
  31#define SWITCHTEC_NTB_MAGIC 0x45CC0001
  32#define MAX_MWS     128
  33
  34struct shared_mw {
  35	u32 magic;
  36	u32 link_sta;
  37	u32 partition_id;
  38	u64 mw_sizes[MAX_MWS];
  39	u32 spad[128];
  40};
  41
  42#define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry)
  43#define LUT_SIZE SZ_64K
  44
  45struct switchtec_ntb {
  46	struct ntb_dev ntb;
  47	struct switchtec_dev *stdev;
  48
  49	int self_partition;
  50	int peer_partition;
  51
  52	int doorbell_irq;
  53	int message_irq;
  54
  55	struct ntb_info_regs __iomem *mmio_ntb;
  56	struct ntb_ctrl_regs __iomem *mmio_ctrl;
  57	struct ntb_dbmsg_regs __iomem *mmio_dbmsg;
  58	struct ntb_ctrl_regs __iomem *mmio_self_ctrl;
  59	struct ntb_ctrl_regs __iomem *mmio_peer_ctrl;
  60	struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg;
  61	struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg;
  62
  63	void __iomem *mmio_xlink_win;
  64
  65	struct shared_mw *self_shared;
  66	struct shared_mw __iomem *peer_shared;
  67	dma_addr_t self_shared_dma;
  68
  69	u64 db_mask;
  70	u64 db_valid_mask;
  71	int db_shift;
  72	int db_peer_shift;
  73
  74	/* synchronize rmw access of db_mask and hw reg */
  75	spinlock_t db_mask_lock;
  76
  77	int nr_direct_mw;
  78	int nr_lut_mw;
  79	int nr_rsvd_luts;
  80	int direct_mw_to_bar[MAX_DIRECT_MW];
  81
  82	int peer_nr_direct_mw;
  83	int peer_nr_lut_mw;
  84	int peer_direct_mw_to_bar[MAX_DIRECT_MW];
  85
  86	bool link_is_up;
  87	enum ntb_speed link_speed;
  88	enum ntb_width link_width;
  89	struct work_struct check_link_status_work;
  90	bool link_force_down;
  91};
  92
  93static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
  94{
  95	return container_of(ntb, struct switchtec_ntb, ntb);
  96}
  97
  98static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
  99				 struct ntb_ctrl_regs __iomem *ctl,
 100				 u32 op, int wait_status)
 101{
 102	static const char * const op_text[] = {
 103		[NTB_CTRL_PART_OP_LOCK] = "lock",
 104		[NTB_CTRL_PART_OP_CFG] = "configure",
 105		[NTB_CTRL_PART_OP_RESET] = "reset",
 106	};
 107
 108	int i;
 109	u32 ps;
 110	int status;
 111
 112	switch (op) {
 113	case NTB_CTRL_PART_OP_LOCK:
 114		status = NTB_CTRL_PART_STATUS_LOCKING;
 115		break;
 116	case NTB_CTRL_PART_OP_CFG:
 117		status = NTB_CTRL_PART_STATUS_CONFIGURING;
 118		break;
 119	case NTB_CTRL_PART_OP_RESET:
 120		status = NTB_CTRL_PART_STATUS_RESETTING;
 121		break;
 122	default:
 123		return -EINVAL;
 124	}
 125
 126	iowrite32(op, &ctl->partition_op);
 127
 128	for (i = 0; i < 1000; i++) {
 129		if (msleep_interruptible(50) != 0) {
 130			iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op);
 131			return -EINTR;
 132		}
 133
 134		ps = ioread32(&ctl->partition_status) & 0xFFFF;
 135
 136		if (ps != status)
 137			break;
 138	}
 139
 140	if (ps == wait_status)
 141		return 0;
 142
 143	if (ps == status) {
 144		dev_err(&sndev->stdev->dev,
 145			"Timed out while performing %s (%d). (%08x)\n",
 146			op_text[op], op,
 147			ioread32(&ctl->partition_status));
 148
 149		return -ETIMEDOUT;
 150	}
 151
 152	return -EIO;
 153}
 154
 155static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,
 156				  u32 val)
 157{
 158	if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))
 159		return -EINVAL;
 160
 161	iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);
 162
 163	return 0;
 164}
 165
 166static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 167{
 168	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 169	int nr_direct_mw = sndev->peer_nr_direct_mw;
 170	int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;
 171
 172	if (pidx != NTB_DEF_PEER_IDX)
 173		return -EINVAL;
 174
 175	if (!use_lut_mws)
 176		nr_lut_mw = 0;
 177
 178	return nr_direct_mw + nr_lut_mw;
 179}
 180
 181static int lut_index(struct switchtec_ntb *sndev, int mw_idx)
 182{
 183	return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;
 184}
 185
 186static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)
 187{
 188	return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;
 189}
 190
 191static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx,
 192				      int widx, resource_size_t *addr_align,
 193				      resource_size_t *size_align,
 194				      resource_size_t *size_max)
 195{
 196	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 197	int lut;
 198	resource_size_t size;
 199
 200	if (pidx != NTB_DEF_PEER_IDX)
 201		return -EINVAL;
 202
 203	lut = widx >= sndev->peer_nr_direct_mw;
 204	size = ioread64(&sndev->peer_shared->mw_sizes[widx]);
 205
 206	if (size == 0)
 207		return -EINVAL;
 208
 209	if (addr_align)
 210		*addr_align = lut ? size : SZ_4K;
 211
 212	if (size_align)
 213		*size_align = lut ? size : SZ_4K;
 214
 215	if (size_max)
 216		*size_max = size;
 217
 218	return 0;
 219}
 220
 221static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx)
 222{
 223	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 224	int bar = sndev->peer_direct_mw_to_bar[idx];
 225	u32 ctl_val;
 226
 227	ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
 228	ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN;
 229	iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
 230	iowrite32(0, &ctl->bar_entry[bar].win_size);
 231	iowrite32(0, &ctl->bar_ext_entry[bar].win_size);
 232	iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr);
 233}
 234
 235static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx)
 236{
 237	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 238
 239	iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]);
 240}
 241
 242static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx,
 243					dma_addr_t addr, resource_size_t size)
 244{
 245	int xlate_pos = ilog2(size);
 246	int bar = sndev->peer_direct_mw_to_bar[idx];
 247	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 248	u32 ctl_val;
 249
 250	ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
 251	ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
 252
 253	iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
 254	iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
 255		  &ctl->bar_entry[bar].win_size);
 256	iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
 257	iowrite64(sndev->self_partition | addr,
 258		  &ctl->bar_entry[bar].xlate_addr);
 259}
 260
 261static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx,
 262				     dma_addr_t addr, resource_size_t size)
 263{
 264	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 265
 266	iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr),
 267		  &ctl->lut_entry[peer_lut_index(sndev, idx)]);
 268}
 269
 270static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
 271				      dma_addr_t addr, resource_size_t size)
 272{
 273	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 274	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 275	int xlate_pos = ilog2(size);
 276	int nr_direct_mw = sndev->peer_nr_direct_mw;
 277	int rc;
 278
 279	if (pidx != NTB_DEF_PEER_IDX)
 280		return -EINVAL;
 281
 282	dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",
 283		widx, pidx, &addr, &size);
 284
 285	if (widx >= switchtec_ntb_mw_count(ntb, pidx))
 286		return -EINVAL;
 287
 288	if (size != 0 && xlate_pos < 12)
 289		return -EINVAL;
 290
 291	if (xlate_pos >= 0 && !IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
 292		/*
 293		 * In certain circumstances we can get a buffer that is
 294		 * not aligned to its size. (Most of the time
 295		 * dma_alloc_coherent ensures this). This can happen when
 296		 * using large buffers allocated by the CMA
 297		 * (see CMA_CONFIG_ALIGNMENT)
 298		 */
 299		dev_err(&sndev->stdev->dev,
 300			"ERROR: Memory window address is not aligned to its size!\n");
 301		return -EINVAL;
 302	}
 303
 304	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
 305				   NTB_CTRL_PART_STATUS_LOCKED);
 306	if (rc)
 307		return rc;
 308
 309	if (size == 0) {
 310		if (widx < nr_direct_mw)
 311			switchtec_ntb_mw_clr_direct(sndev, widx);
 312		else
 313			switchtec_ntb_mw_clr_lut(sndev, widx);
 314	} else {
 315		if (widx < nr_direct_mw)
 316			switchtec_ntb_mw_set_direct(sndev, widx, addr, size);
 317		else
 318			switchtec_ntb_mw_set_lut(sndev, widx, addr, size);
 319	}
 320
 321	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
 322				   NTB_CTRL_PART_STATUS_NORMAL);
 323
 324	if (rc == -EIO) {
 325		dev_err(&sndev->stdev->dev,
 326			"Hardware reported an error configuring mw %d: %08x\n",
 327			widx, ioread32(&ctl->bar_error));
 328
 329		if (widx < nr_direct_mw)
 330			switchtec_ntb_mw_clr_direct(sndev, widx);
 331		else
 332			switchtec_ntb_mw_clr_lut(sndev, widx);
 333
 334		switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
 335				      NTB_CTRL_PART_STATUS_NORMAL);
 336	}
 337
 338	return rc;
 339}
 340
 341static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb)
 342{
 343	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 344	int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;
 345
 346	return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);
 347}
 348
 349static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,
 350					 int idx, phys_addr_t *base,
 351					 resource_size_t *size)
 352{
 353	int bar = sndev->direct_mw_to_bar[idx];
 354	size_t offset = 0;
 355
 356	if (bar < 0)
 357		return -EINVAL;
 358
 359	if (idx == 0) {
 360		/*
 361		 * This is the direct BAR shared with the LUTs
 362		 * which means the actual window will be offset
 363		 * by the size of all the LUT entries.
 364		 */
 365
 366		offset = LUT_SIZE * sndev->nr_lut_mw;
 367	}
 368
 369	if (base)
 370		*base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
 371
 372	if (size) {
 373		*size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
 374		if (offset && *size > offset)
 375			*size = offset;
 376
 377		if (*size > max_mw_size)
 378			*size = max_mw_size;
 379	}
 380
 381	return 0;
 382}
 383
 384static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev,
 385				      int idx, phys_addr_t *base,
 386				      resource_size_t *size)
 387{
 388	int bar = sndev->direct_mw_to_bar[0];
 389	int offset;
 390
 391	offset = LUT_SIZE * lut_index(sndev, idx);
 392
 393	if (base)
 394		*base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
 395
 396	if (size)
 397		*size = LUT_SIZE;
 398
 399	return 0;
 400}
 401
 402static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
 403					  phys_addr_t *base,
 404					  resource_size_t *size)
 405{
 406	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 407
 408	if (idx < sndev->nr_direct_mw)
 409		return switchtec_ntb_direct_get_addr(sndev, idx, base, size);
 410	else if (idx < switchtec_ntb_peer_mw_count(ntb))
 411		return switchtec_ntb_lut_get_addr(sndev, idx, base, size);
 412	else
 413		return -EINVAL;
 414}
 415
 416static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev,
 417					  int partition,
 418					  enum ntb_speed *speed,
 419					  enum ntb_width *width)
 420{
 421	struct switchtec_dev *stdev = sndev->stdev;
 422	struct part_cfg_regs __iomem *part_cfg =
 423		&stdev->mmio_part_cfg_all[partition];
 424
 425	u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
 426	u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]);
 427
 428	if (speed)
 429		*speed = (linksta >> 16) & 0xF;
 430
 431	if (width)
 432		*width = (linksta >> 20) & 0x3F;
 433}
 434
 435static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)
 436{
 437	enum ntb_speed self_speed, peer_speed;
 438	enum ntb_width self_width, peer_width;
 439
 440	if (!sndev->link_is_up) {
 441		sndev->link_speed = NTB_SPEED_NONE;
 442		sndev->link_width = NTB_WIDTH_NONE;
 443		return;
 444	}
 445
 446	switchtec_ntb_part_link_speed(sndev, sndev->self_partition,
 447				      &self_speed, &self_width);
 448	switchtec_ntb_part_link_speed(sndev, sndev->peer_partition,
 449				      &peer_speed, &peer_width);
 450
 451	sndev->link_speed = min(self_speed, peer_speed);
 452	sndev->link_width = min(self_width, peer_width);
 453}
 454
 455static int crosslink_is_enabled(struct switchtec_ntb *sndev)
 456{
 457	struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;
 458
 459	return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);
 460}
 461
 462static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)
 463{
 464	int i;
 465	u32 msg_map = 0;
 466
 467	if (!crosslink_is_enabled(sndev))
 468		return;
 469
 470	for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {
 471		int m = i | sndev->self_partition << 2;
 472
 473		msg_map |= m << i * 8;
 474	}
 475
 476	iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);
 477	iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
 478		  &sndev->mmio_peer_dbmsg->odb_mask);
 479}
 480
 481enum switchtec_msg {
 482	LINK_MESSAGE = 0,
 483	MSG_LINK_UP = 1,
 484	MSG_LINK_DOWN = 2,
 485	MSG_CHECK_LINK = 3,
 486	MSG_LINK_FORCE_DOWN = 4,
 487};
 488
 489static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
 490
 491static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev)
 492{
 493	int link_sta;
 494	int old = sndev->link_is_up;
 495
 496	link_sta = sndev->self_shared->link_sta;
 497	if (link_sta) {
 498		u64 peer = ioread64(&sndev->peer_shared->magic);
 499
 500		if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC)
 501			link_sta = peer >> 32;
 502		else
 503			link_sta = 0;
 504	}
 505
 506	sndev->link_is_up = link_sta;
 507	switchtec_ntb_set_link_speed(sndev);
 508
 509	if (link_sta != old) {
 510		switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK);
 511		ntb_link_event(&sndev->ntb);
 512		dev_info(&sndev->stdev->dev, "ntb link %s\n",
 513			 link_sta ? "up" : "down");
 514
 515		if (link_sta)
 516			crosslink_init_dbmsgs(sndev);
 517	}
 518}
 519
 520static void check_link_status_work(struct work_struct *work)
 521{
 522	struct switchtec_ntb *sndev;
 523
 524	sndev = container_of(work, struct switchtec_ntb,
 525			     check_link_status_work);
 526
 527	if (sndev->link_force_down) {
 528		sndev->link_force_down = false;
 529		switchtec_ntb_reinit_peer(sndev);
 530
 531		if (sndev->link_is_up) {
 532			sndev->link_is_up = 0;
 533			ntb_link_event(&sndev->ntb);
 534			dev_info(&sndev->stdev->dev, "ntb link forced down\n");
 535		}
 536
 537		return;
 538	}
 539
 540	switchtec_ntb_link_status_update(sndev);
 541}
 542
 543static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
 544				      enum switchtec_msg msg)
 545{
 546	if (msg == MSG_LINK_FORCE_DOWN)
 547		sndev->link_force_down = true;
 548
 549	schedule_work(&sndev->check_link_status_work);
 550}
 551
 552static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
 553{
 554	struct switchtec_ntb *sndev = stdev->sndev;
 555
 556	switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
 557}
 558
 559static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb,
 560				    enum ntb_speed *speed,
 561				    enum ntb_width *width)
 562{
 563	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 564
 565	if (speed)
 566		*speed = sndev->link_speed;
 567	if (width)
 568		*width = sndev->link_width;
 569
 570	return sndev->link_is_up;
 571}
 572
 573static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
 574				     enum ntb_speed max_speed,
 575				     enum ntb_width max_width)
 576{
 577	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 578
 579	dev_dbg(&sndev->stdev->dev, "enabling link\n");
 580
 581	sndev->self_shared->link_sta = 1;
 582	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);
 583
 584	switchtec_ntb_link_status_update(sndev);
 585
 586	return 0;
 587}
 588
 589static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
 590{
 591	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 592
 593	dev_dbg(&sndev->stdev->dev, "disabling link\n");
 594
 595	sndev->self_shared->link_sta = 0;
 596	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN);
 597
 598	switchtec_ntb_link_status_update(sndev);
 599
 600	return 0;
 601}
 602
 603static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb)
 604{
 605	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 606
 607	return sndev->db_valid_mask;
 608}
 609
 610static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb)
 611{
 612	return 1;
 613}
 614
 615static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
 616{
 617	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 618
 619	if (db_vector < 0 || db_vector > 1)
 620		return 0;
 621
 622	return sndev->db_valid_mask;
 623}
 624
 625static u64 switchtec_ntb_db_read(struct ntb_dev *ntb)
 626{
 627	u64 ret;
 628	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 629
 630	ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift;
 631
 632	return ret & sndev->db_valid_mask;
 633}
 634
 635static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
 636{
 637	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 638
 639	iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb);
 640
 641	return 0;
 642}
 643
 644static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
 645{
 646	unsigned long irqflags;
 647	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 648
 649	if (db_bits & ~sndev->db_valid_mask)
 650		return -EINVAL;
 651
 652	spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
 653
 654	sndev->db_mask |= db_bits << sndev->db_shift;
 655	iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
 656
 657	spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
 658
 659	return 0;
 660}
 661
 662static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
 663{
 664	unsigned long irqflags;
 665	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 666
 667	if (db_bits & ~sndev->db_valid_mask)
 668		return -EINVAL;
 669
 670	spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
 671
 672	sndev->db_mask &= ~(db_bits << sndev->db_shift);
 673	iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
 674
 675	spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
 676
 677	return 0;
 678}
 679
 680static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb)
 681{
 682	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 683
 684	return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask;
 685}
 686
 687static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
 688				      phys_addr_t *db_addr,
 689				      resource_size_t *db_size,
 690				      u64 *db_data,
 691				      int db_bit)
 692{
 693	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 694	unsigned long offset;
 695
 696	if (unlikely(db_bit >= BITS_PER_LONG_LONG))
 697		return -EINVAL;
 698
 699	offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
 700		(unsigned long)sndev->stdev->mmio;
 701
 702	offset += sndev->db_shift / 8;
 703
 704	if (db_addr)
 705		*db_addr = pci_resource_start(ntb->pdev, 0) + offset;
 706	if (db_size)
 707		*db_size = sizeof(u32);
 708	if (db_data)
 709		*db_data = BIT_ULL(db_bit) << sndev->db_peer_shift;
 710
 711	return 0;
 712}
 713
 714static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
 715{
 716	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 717
 718	iowrite64(db_bits << sndev->db_peer_shift,
 719		  &sndev->mmio_peer_dbmsg->odb);
 720
 721	return 0;
 722}
 723
 724static int switchtec_ntb_spad_count(struct ntb_dev *ntb)
 725{
 726	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 727
 728	return ARRAY_SIZE(sndev->self_shared->spad);
 729}
 730
 731static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx)
 732{
 733	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 734
 735	if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
 736		return 0;
 737
 738	if (!sndev->self_shared)
 739		return 0;
 740
 741	return sndev->self_shared->spad[idx];
 742}
 743
 744static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
 745{
 746	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 747
 748	if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
 749		return -EINVAL;
 750
 751	if (!sndev->self_shared)
 752		return -EIO;
 753
 754	sndev->self_shared->spad[idx] = val;
 755
 756	return 0;
 757}
 758
 759static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx,
 760					int sidx)
 761{
 762	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 763
 764	if (pidx != NTB_DEF_PEER_IDX)
 765		return -EINVAL;
 766
 767	if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
 768		return 0;
 769
 770	if (!sndev->peer_shared)
 771		return 0;
 772
 773	return ioread32(&sndev->peer_shared->spad[sidx]);
 774}
 775
 776static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
 777					 int sidx, u32 val)
 778{
 779	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 780
 781	if (pidx != NTB_DEF_PEER_IDX)
 782		return -EINVAL;
 783
 784	if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
 785		return -EINVAL;
 786
 787	if (!sndev->peer_shared)
 788		return -EIO;
 789
 790	iowrite32(val, &sndev->peer_shared->spad[sidx]);
 791
 792	return 0;
 793}
 794
 795static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx,
 796					int sidx, phys_addr_t *spad_addr)
 797{
 798	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 799	unsigned long offset;
 800
 801	if (pidx != NTB_DEF_PEER_IDX)
 802		return -EINVAL;
 803
 804	offset = (unsigned long)&sndev->peer_shared->spad[sidx] -
 805		(unsigned long)sndev->stdev->mmio;
 806
 807	if (spad_addr)
 808		*spad_addr = pci_resource_start(ntb->pdev, 0) + offset;
 809
 810	return 0;
 811}
 812
 813static const struct ntb_dev_ops switchtec_ntb_ops = {
 814	.mw_count		= switchtec_ntb_mw_count,
 815	.mw_get_align		= switchtec_ntb_mw_get_align,
 816	.mw_set_trans		= switchtec_ntb_mw_set_trans,
 817	.peer_mw_count		= switchtec_ntb_peer_mw_count,
 818	.peer_mw_get_addr	= switchtec_ntb_peer_mw_get_addr,
 819	.link_is_up		= switchtec_ntb_link_is_up,
 820	.link_enable		= switchtec_ntb_link_enable,
 821	.link_disable		= switchtec_ntb_link_disable,
 822	.db_valid_mask		= switchtec_ntb_db_valid_mask,
 823	.db_vector_count	= switchtec_ntb_db_vector_count,
 824	.db_vector_mask		= switchtec_ntb_db_vector_mask,
 825	.db_read		= switchtec_ntb_db_read,
 826	.db_clear		= switchtec_ntb_db_clear,
 827	.db_set_mask		= switchtec_ntb_db_set_mask,
 828	.db_clear_mask		= switchtec_ntb_db_clear_mask,
 829	.db_read_mask		= switchtec_ntb_db_read_mask,
 830	.peer_db_addr		= switchtec_ntb_peer_db_addr,
 831	.peer_db_set		= switchtec_ntb_peer_db_set,
 832	.spad_count		= switchtec_ntb_spad_count,
 833	.spad_read		= switchtec_ntb_spad_read,
 834	.spad_write		= switchtec_ntb_spad_write,
 835	.peer_spad_read		= switchtec_ntb_peer_spad_read,
 836	.peer_spad_write	= switchtec_ntb_peer_spad_write,
 837	.peer_spad_addr		= switchtec_ntb_peer_spad_addr,
 838};
 839
 840static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
 841{
 842	u64 tpart_vec;
 843	int self;
 844	u64 part_map;
 845
 846	sndev->ntb.pdev = sndev->stdev->pdev;
 847	sndev->ntb.topo = NTB_TOPO_SWITCH;
 848	sndev->ntb.ops = &switchtec_ntb_ops;
 849
 850	INIT_WORK(&sndev->check_link_status_work, check_link_status_work);
 851	sndev->link_force_down = false;
 852
 853	sndev->self_partition = sndev->stdev->partition;
 854
 855	sndev->mmio_ntb = sndev->stdev->mmio_ntb;
 856
 857	self = sndev->self_partition;
 858	tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);
 859	tpart_vec <<= 32;
 860	tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);
 861
 862	part_map = ioread64(&sndev->mmio_ntb->ep_map);
 863	tpart_vec &= part_map;
 864	part_map &= ~(1 << sndev->self_partition);
 865
 866	if (!tpart_vec) {
 867		if (sndev->stdev->partition_count != 2) {
 868			dev_err(&sndev->stdev->dev,
 869				"ntb target partition not defined\n");
 870			return -ENODEV;
 871		}
 872
 873		if (!part_map) {
 874			dev_err(&sndev->stdev->dev,
 875				"peer partition is not NT partition\n");
 876			return -ENODEV;
 877		}
 878
 879		sndev->peer_partition = __ffs64(part_map);
 880	} else {
 881		if (__ffs64(tpart_vec) != (fls64(tpart_vec) - 1)) {
 882			dev_err(&sndev->stdev->dev,
 883				"ntb driver only supports 1 pair of 1-1 ntb mapping\n");
 884			return -ENODEV;
 885		}
 886
 887		sndev->peer_partition = __ffs64(tpart_vec);
 888		if (!(part_map & (1ULL << sndev->peer_partition))) {
 889			dev_err(&sndev->stdev->dev,
 890				"ntb target partition is not NT partition\n");
 891			return -ENODEV;
 892		}
 893	}
 894
 895	dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",
 896		sndev->self_partition, sndev->stdev->partition_count);
 897
 898	sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +
 899		SWITCHTEC_NTB_REG_CTRL_OFFSET;
 900	sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb +
 901		SWITCHTEC_NTB_REG_DBMSG_OFFSET;
 902
 903	sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];
 904	sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];
 905	sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];
 906	sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;
 907
 908	return 0;
 909}
 910
 911static int config_rsvd_lut_win(struct switchtec_ntb *sndev,
 912			       struct ntb_ctrl_regs __iomem *ctl,
 913			       int lut_idx, int partition, u64 addr)
 914{
 915	int peer_bar = sndev->peer_direct_mw_to_bar[0];
 916	u32 ctl_val;
 917	int rc;
 918
 919	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
 920				   NTB_CTRL_PART_STATUS_LOCKED);
 921	if (rc)
 922		return rc;
 923
 924	ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl);
 925	ctl_val &= 0xFF;
 926	ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
 927	ctl_val |= ilog2(LUT_SIZE) << 8;
 928	ctl_val |= (sndev->nr_lut_mw - 1) << 14;
 929	iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl);
 930
 931	iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr),
 932		  &ctl->lut_entry[lut_idx]);
 933
 934	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
 935				   NTB_CTRL_PART_STATUS_NORMAL);
 936	if (rc) {
 937		u32 bar_error, lut_error;
 938
 939		bar_error = ioread32(&ctl->bar_error);
 940		lut_error = ioread32(&ctl->lut_error);
 941		dev_err(&sndev->stdev->dev,
 942			"Error setting up reserved lut window: %08x / %08x\n",
 943			bar_error, lut_error);
 944		return rc;
 945	}
 946
 947	return 0;
 948}
 949
 950static int config_req_id_table(struct switchtec_ntb *sndev,
 951			       struct ntb_ctrl_regs __iomem *mmio_ctrl,
 952			       int *req_ids, int count)
 953{
 954	int i, rc = 0;
 955	u32 error;
 956	u32 proxy_id;
 957
 958	if (ioread16(&mmio_ctrl->req_id_table_size) < count) {
 959		dev_err(&sndev->stdev->dev,
 960			"Not enough requester IDs available.\n");
 961		return -EFAULT;
 962	}
 963
 964	rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
 965				   NTB_CTRL_PART_OP_LOCK,
 966				   NTB_CTRL_PART_STATUS_LOCKED);
 967	if (rc)
 968		return rc;
 969
 970	for (i = 0; i < count; i++) {
 971		iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN,
 972			  &mmio_ctrl->req_id_table[i]);
 973
 974		proxy_id = ioread32(&mmio_ctrl->req_id_table[i]);
 975		dev_dbg(&sndev->stdev->dev,
 976			"Requester ID %02X:%02X.%X -> BB:%02X.%X\n",
 977			req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F,
 978			req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F,
 979			(proxy_id >> 1) & 0x7);
 980	}
 981
 982	rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
 983				   NTB_CTRL_PART_OP_CFG,
 984				   NTB_CTRL_PART_STATUS_NORMAL);
 985
 986	if (rc == -EIO) {
 987		error = ioread32(&mmio_ctrl->req_id_error);
 988		dev_err(&sndev->stdev->dev,
 989			"Error setting up the requester ID table: %08x\n",
 990			error);
 991	}
 992
 993	return 0;
 994}
 995
 996static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,
 997			       u64 *mw_addrs, int mw_count)
 998{
 999	int rc, i;
1000	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;
1001	u64 addr;
1002	size_t size, offset;
1003	int bar;
1004	int xlate_pos;
1005	u32 ctl_val;
1006
1007	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
1008				   NTB_CTRL_PART_STATUS_LOCKED);
1009	if (rc)
1010		return rc;
1011
1012	for (i = 0; i < sndev->nr_lut_mw; i++) {
1013		if (i == ntb_lut_idx)
1014			continue;
1015
1016		addr = mw_addrs[0] + LUT_SIZE * i;
1017
1018		iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |
1019			   addr),
1020			  &ctl->lut_entry[i]);
1021	}
1022
1023	sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);
1024
1025	for (i = 0; i < sndev->nr_direct_mw; i++) {
1026		bar = sndev->direct_mw_to_bar[i];
1027		offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;
1028		addr = mw_addrs[i] + offset;
1029		size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
1030		xlate_pos = ilog2(size);
1031
1032		if (offset && size > offset)
1033			size = offset;
1034
1035		ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
1036		ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
1037
1038		iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
1039		iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
1040			  &ctl->bar_entry[bar].win_size);
1041		iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
1042		iowrite64(sndev->peer_partition | addr,
1043			  &ctl->bar_entry[bar].xlate_addr);
1044	}
1045
1046	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
1047				   NTB_CTRL_PART_STATUS_NORMAL);
1048	if (rc) {
1049		u32 bar_error, lut_error;
1050
1051		bar_error = ioread32(&ctl->bar_error);
1052		lut_error = ioread32(&ctl->lut_error);
1053		dev_err(&sndev->stdev->dev,
1054			"Error setting up cross link windows: %08x / %08x\n",
1055			bar_error, lut_error);
1056		return rc;
1057	}
1058
1059	return 0;
1060}
1061
1062static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,
1063	struct ntb_ctrl_regs __iomem *mmio_ctrl)
1064{
1065	int req_ids[16];
1066	int i;
1067	u32 proxy_id;
1068
1069	for (i = 0; i < ARRAY_SIZE(req_ids); i++) {
1070		proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);
1071
1072		if (!(proxy_id & NTB_CTRL_REQ_ID_EN))
1073			break;
1074
1075		req_ids[i] = ((proxy_id >> 1) & 0xFF);
1076	}
1077
1078	return config_req_id_table(sndev, mmio_ctrl, req_ids, i);
1079}
1080
1081/*
1082 * In crosslink configuration there is a virtual partition in the
1083 * middle of the two switches. The BARs in this partition have to be
1084 * enumerated and assigned addresses.
1085 */
1086static int crosslink_enum_partition(struct switchtec_ntb *sndev,
1087				    u64 *bar_addrs)
1088{
1089	struct part_cfg_regs __iomem *part_cfg =
1090		&sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];
1091	u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
1092	struct pff_csr_regs __iomem *mmio_pff =
1093		&sndev->stdev->mmio_pff_csr[pff];
1094	const u64 bar_space = 0x1000000000LL;
1095	u64 bar_addr;
1096	int bar_cnt = 0;
1097	int i;
1098
1099	iowrite16(0x6, &mmio_pff->pcicmd);
1100
1101	for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) {
1102		iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]);
1103		bar_addr = ioread64(&mmio_pff->pci_bar64[i]);
1104		bar_addr &= ~0xf;
1105
1106		dev_dbg(&sndev->stdev->dev,
1107			"Crosslink BAR%d addr: %llx\n",
1108			i*2, bar_addr);
1109
1110		if (bar_addr != bar_space * i)
1111			continue;
1112
1113		bar_addrs[bar_cnt++] = bar_addr;
1114	}
1115
1116	return bar_cnt;
1117}
1118
1119static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)
1120{
1121	int rc;
1122	int bar = sndev->direct_mw_to_bar[0];
1123	const int ntb_lut_idx = 1;
1124	u64 bar_addrs[6];
1125	u64 addr;
1126	int offset;
1127	int bar_cnt;
1128
1129	if (!crosslink_is_enabled(sndev))
1130		return 0;
1131
1132	dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");
1133	sndev->ntb.topo = NTB_TOPO_CROSSLINK;
1134
1135	bar_cnt = crosslink_enum_partition(sndev, bar_addrs);
1136	if (bar_cnt < sndev->nr_direct_mw + 1) {
1137		dev_err(&sndev->stdev->dev,
1138			"Error enumerating crosslink partition\n");
1139		return -EINVAL;
1140	}
1141
1142	addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET +
1143		SWITCHTEC_NTB_REG_DBMSG_OFFSET +
1144		sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);
1145
1146	offset = addr & (LUT_SIZE - 1);
1147	addr -= offset;
1148
1149	rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx,
1150				 sndev->peer_partition, addr);
1151	if (rc)
1152		return rc;
1153
1154	rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1],
1155				 bar_cnt - 1);
1156	if (rc)
1157		return rc;
1158
1159	rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl);
1160	if (rc)
1161		return rc;
1162
1163	sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar,
1164						LUT_SIZE, LUT_SIZE);
1165	if (!sndev->mmio_xlink_win) {
1166		rc = -ENOMEM;
1167		return rc;
1168	}
1169
1170	sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;
1171	sndev->nr_rsvd_luts++;
1172
1173	crosslink_init_dbmsgs(sndev);
1174
1175	return 0;
1176}
1177
1178static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)
1179{
1180	if (sndev->mmio_xlink_win)
1181		pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win);
1182}
1183
1184static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl)
1185{
1186	int i;
1187	int cnt = 0;
1188
1189	for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) {
1190		u32 r = ioread32(&ctrl->bar_entry[i].ctl);
1191
1192		if (r & NTB_CTRL_BAR_VALID)
1193			map[cnt++] = i;
1194	}
1195
1196	return cnt;
1197}
1198
1199static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
1200{
1201	sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar,
1202				       sndev->mmio_self_ctrl);
1203
1204	sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
1205	if (sndev->nr_lut_mw)
1206		sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
1207
1208	dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
1209		sndev->nr_direct_mw, sndev->nr_lut_mw);
1210
1211	sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar,
1212					    sndev->mmio_peer_ctrl);
1213
1214	sndev->peer_nr_lut_mw =
1215		ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
1216	if (sndev->peer_nr_lut_mw)
1217		sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
1218
1219	dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
1220		sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
1221
1222}
1223
1224/*
1225 * There are 64 doorbells in the switch hardware but this is
1226 * shared among all partitions. So we must split them in half
1227 * (32 for each partition). However, the message interrupts are
1228 * also shared with the top 4 doorbells so we just limit this to
1229 * 28 doorbells per partition.
1230 *
1231 * In crosslink mode, each side has it's own dbmsg register so
1232 * they can each use all 60 of the available doorbells.
1233 */
1234static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)
1235{
1236	sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
1237
1238	if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {
1239		sndev->db_shift = 0;
1240		sndev->db_peer_shift = 0;
1241		sndev->db_valid_mask = sndev->db_mask;
1242	} else if (sndev->self_partition < sndev->peer_partition) {
1243		sndev->db_shift = 0;
1244		sndev->db_peer_shift = 32;
1245		sndev->db_valid_mask = 0x0FFFFFFF;
1246	} else {
1247		sndev->db_shift = 32;
1248		sndev->db_peer_shift = 0;
1249		sndev->db_valid_mask = 0x0FFFFFFF;
1250	}
1251
1252	iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
1253	iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
1254		  &sndev->mmio_peer_dbmsg->odb_mask);
1255
1256	dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",
1257		sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);
1258}
1259
1260static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
1261{
1262	int i;
1263	u32 msg_map = 0;
1264
1265	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1266		int m = i | sndev->peer_partition << 2;
1267
1268		msg_map |= m << i * 8;
1269	}
1270
1271	iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map);
1272
1273	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++)
1274		iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK,
1275			  &sndev->mmio_self_dbmsg->imsg[i]);
1276}
1277
1278static int
1279switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
1280{
1281	int req_ids[2];
1282
1283	/*
1284	 * Root Complex Requester ID (which is 0:00.0)
1285	 */
1286	req_ids[0] = 0;
1287
1288	/*
1289	 * Host Bridge Requester ID (as read from the mmap address)
1290	 */
1291	req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);
1292
1293	return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids,
1294				   ARRAY_SIZE(req_ids));
1295}
1296
1297static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
1298{
1299	int i;
1300
1301	memset(sndev->self_shared, 0, LUT_SIZE);
1302	sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC;
1303	sndev->self_shared->partition_id = sndev->stdev->partition;
1304
1305	for (i = 0; i < sndev->nr_direct_mw; i++) {
1306		int bar = sndev->direct_mw_to_bar[i];
1307		resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar);
1308
1309		if (i == 0)
1310			sz = min_t(resource_size_t, sz,
1311				   LUT_SIZE * sndev->nr_lut_mw);
1312
1313		sndev->self_shared->mw_sizes[i] = sz;
1314	}
1315
1316	for (i = 0; i < sndev->nr_lut_mw; i++) {
1317		int idx = sndev->nr_direct_mw + i;
1318
1319		if (idx >= MAX_MWS) {
1320			dev_err(&sndev->stdev->dev,
1321				"Total number of MW cannot be bigger than %d", MAX_MWS);
1322			break;
1323		}
1324
1325		sndev->self_shared->mw_sizes[idx] = LUT_SIZE;
1326	}
1327}
1328
1329static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)
1330{
1331	int self_bar = sndev->direct_mw_to_bar[0];
1332	int rc;
1333
1334	sndev->nr_rsvd_luts++;
1335	sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev,
1336						LUT_SIZE,
1337						&sndev->self_shared_dma,
1338						GFP_KERNEL);
1339	if (!sndev->self_shared) {
1340		dev_err(&sndev->stdev->dev,
1341			"unable to allocate memory for shared mw\n");
1342		return -ENOMEM;
1343	}
1344
1345	switchtec_ntb_init_shared(sndev);
1346
1347	rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
1348				 sndev->self_partition,
1349				 sndev->self_shared_dma);
1350	if (rc)
1351		goto unalloc_and_exit;
1352
1353	sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE);
1354	if (!sndev->peer_shared) {
1355		rc = -ENOMEM;
1356		goto unalloc_and_exit;
1357	}
1358
1359	dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");
1360	return 0;
1361
1362unalloc_and_exit:
1363	dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1364			  sndev->self_shared, sndev->self_shared_dma);
1365
1366	return rc;
1367}
1368
1369static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)
1370{
1371	if (sndev->peer_shared)
1372		pci_iounmap(sndev->stdev->pdev, sndev->peer_shared);
1373
1374	if (sndev->self_shared)
1375		dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1376				  sndev->self_shared,
1377				  sndev->self_shared_dma);
1378	sndev->nr_rsvd_luts--;
1379}
1380
1381static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev)
1382{
1383	struct switchtec_ntb *sndev = dev;
1384
1385	dev_dbg(&sndev->stdev->dev, "doorbell\n");
1386
1387	ntb_db_event(&sndev->ntb, 0);
1388
1389	return IRQ_HANDLED;
1390}
1391
1392static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev)
1393{
1394	int i;
1395	struct switchtec_ntb *sndev = dev;
1396
1397	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1398		u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]);
1399
1400		if (msg & NTB_DBMSG_IMSG_STATUS) {
1401			dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",
1402				i, (u32)msg);
1403			iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);
1404
1405			if (i == LINK_MESSAGE)
1406				switchtec_ntb_check_link(sndev, msg);
1407		}
1408	}
1409
1410	return IRQ_HANDLED;
1411}
1412
1413static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)
1414{
1415	int i;
1416	int rc;
1417	int doorbell_irq = 0;
1418	int message_irq = 0;
1419	int event_irq;
1420	int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map);
1421
1422	event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number);
1423
1424	while (doorbell_irq == event_irq)
1425		doorbell_irq++;
1426	while (message_irq == doorbell_irq ||
1427	       message_irq == event_irq)
1428		message_irq++;
1429
1430	dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",
1431		event_irq, doorbell_irq, message_irq);
1432
1433	for (i = 0; i < idb_vecs - 4; i++)
1434		iowrite8(doorbell_irq,
1435			 &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1436
1437	for (; i < idb_vecs; i++)
1438		iowrite8(message_irq,
1439			 &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1440
1441	sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq);
1442	sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq);
1443
1444	rc = request_irq(sndev->doorbell_irq,
1445			 switchtec_ntb_doorbell_isr, 0,
1446			 "switchtec_ntb_doorbell", sndev);
1447	if (rc)
1448		return rc;
1449
1450	rc = request_irq(sndev->message_irq,
1451			 switchtec_ntb_message_isr, 0,
1452			 "switchtec_ntb_message", sndev);
1453	if (rc) {
1454		free_irq(sndev->doorbell_irq, sndev);
1455		return rc;
1456	}
1457
1458	return 0;
1459}
1460
1461static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
1462{
1463	free_irq(sndev->doorbell_irq, sndev);
1464	free_irq(sndev->message_irq, sndev);
1465}
1466
1467static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
1468{
1469	int rc;
1470
1471	if (crosslink_is_enabled(sndev))
1472		return 0;
1473
1474	dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n");
1475	rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
1476				 sndev->self_partition,
1477				 sndev->self_shared_dma);
1478	return rc;
1479}
1480
1481static int switchtec_ntb_add(struct device *dev)
1482{
1483	struct switchtec_dev *stdev = to_stdev(dev);
1484	struct switchtec_ntb *sndev;
1485	int rc;
1486
1487	stdev->sndev = NULL;
1488
1489	if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8))
1490		return -ENODEV;
1491
1492	sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev));
1493	if (!sndev)
1494		return -ENOMEM;
1495
1496	sndev->stdev = stdev;
1497	rc = switchtec_ntb_init_sndev(sndev);
1498	if (rc)
1499		goto free_and_exit;
1500
1501	switchtec_ntb_init_mw(sndev);
1502
1503	rc = switchtec_ntb_init_req_id_table(sndev);
1504	if (rc)
1505		goto free_and_exit;
1506
1507	rc = switchtec_ntb_init_crosslink(sndev);
1508	if (rc)
1509		goto free_and_exit;
1510
1511	switchtec_ntb_init_db(sndev);
1512	switchtec_ntb_init_msgs(sndev);
1513
1514	rc = switchtec_ntb_init_shared_mw(sndev);
1515	if (rc)
1516		goto deinit_crosslink;
1517
1518	rc = switchtec_ntb_init_db_msg_irq(sndev);
1519	if (rc)
1520		goto deinit_shared_and_exit;
1521
1522	/*
1523	 * If this host crashed, the other host may think the link is
1524	 * still up. Tell them to force it down (it will go back up
1525	 * once we register the ntb device).
1526	 */
1527	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN);
1528
1529	rc = ntb_register_device(&sndev->ntb);
1530	if (rc)
1531		goto deinit_and_exit;
1532
1533	stdev->sndev = sndev;
1534	stdev->link_notifier = switchtec_ntb_link_notification;
1535	dev_info(dev, "NTB device registered\n");
1536
1537	return 0;
1538
1539deinit_and_exit:
1540	switchtec_ntb_deinit_db_msg_irq(sndev);
1541deinit_shared_and_exit:
1542	switchtec_ntb_deinit_shared_mw(sndev);
1543deinit_crosslink:
1544	switchtec_ntb_deinit_crosslink(sndev);
1545free_and_exit:
1546	kfree(sndev);
1547	dev_err(dev, "failed to register ntb device: %d\n", rc);
1548	return rc;
1549}
1550
1551static void switchtec_ntb_remove(struct device *dev)
1552{
1553	struct switchtec_dev *stdev = to_stdev(dev);
1554	struct switchtec_ntb *sndev = stdev->sndev;
1555
1556	if (!sndev)
1557		return;
1558
1559	stdev->link_notifier = NULL;
1560	stdev->sndev = NULL;
1561	ntb_unregister_device(&sndev->ntb);
1562	switchtec_ntb_deinit_db_msg_irq(sndev);
1563	switchtec_ntb_deinit_shared_mw(sndev);
1564	switchtec_ntb_deinit_crosslink(sndev);
1565	cancel_work_sync(&sndev->check_link_status_work);
1566	kfree(sndev);
1567	dev_info(dev, "ntb device unregistered\n");
1568}
1569
1570static struct class_interface switchtec_interface  = {
1571	.add_dev = switchtec_ntb_add,
1572	.remove_dev = switchtec_ntb_remove,
1573};
1574
1575static int __init switchtec_ntb_init(void)
1576{
1577	switchtec_interface.class = &switchtec_class;
1578	return class_interface_register(&switchtec_interface);
1579}
1580module_init(switchtec_ntb_init);
1581
1582static void __exit switchtec_ntb_exit(void)
1583{
1584	class_interface_unregister(&switchtec_interface);
1585}
1586module_exit(switchtec_ntb_exit);