ntb_hw_amd.c

   1/*
   2 * This file is provided under a dual BSD/GPLv2 license.  When using or
   3 *   redistributing this file, you may do so under either license.
   4 *
   5 *   GPL LICENSE SUMMARY
   6 *
   7 *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
   8 *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
   9 *
  10 *   This program is free software; you can redistribute it and/or modify
  11 *   it under the terms of version 2 of the GNU General Public License as
  12 *   published by the Free Software Foundation.
  13 *
  14 *   BSD LICENSE
  15 *
  16 *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
  17 *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  18 *
  19 *   Redistribution and use in source and binary forms, with or without
  20 *   modification, are permitted provided that the following conditions
  21 *   are met:
  22 *
  23 *     * Redistributions of source code must retain the above copyright
  24 *       notice, this list of conditions and the following disclaimer.
  25 *     * Redistributions in binary form must reproduce the above copy
  26 *       notice, this list of conditions and the following disclaimer in
  27 *       the documentation and/or other materials provided with the
  28 *       distribution.
  29 *     * Neither the name of AMD Corporation nor the names of its
  30 *       contributors may be used to endorse or promote products derived
  31 *       from this software without specific prior written permission.
  32 *
  33 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  34 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  35 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  36 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  37 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  38 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  39 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  40 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  41 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  42 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  43 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  44 *
  45 * AMD PCIe NTB Linux driver
  46 *
  47 * Contact Information:
  48 * Xiangliang Yu <Xiangliang.Yu@amd.com>
  49 */
  50
  51#include <linux/debugfs.h>
  52#include <linux/delay.h>
  53#include <linux/init.h>
  54#include <linux/interrupt.h>
  55#include <linux/module.h>
  56#include <linux/acpi.h>
  57#include <linux/pci.h>
  58#include <linux/random.h>
  59#include <linux/slab.h>
  60#include <linux/ntb.h>
  61
  62#include "ntb_hw_amd.h"
  63
  64#define NTB_NAME	"ntb_hw_amd"
  65#define NTB_DESC	"AMD(R) PCI-E Non-Transparent Bridge Driver"
  66#define NTB_VER		"1.0"
  67
  68MODULE_DESCRIPTION(NTB_DESC);
  69MODULE_VERSION(NTB_VER);
  70MODULE_LICENSE("Dual BSD/GPL");
  71MODULE_AUTHOR("AMD Inc.");
  72
  73static const struct file_operations amd_ntb_debugfs_info;
  74static struct dentry *debugfs_dir;
  75
  76static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
  77{
  78	if (idx < 0 || idx > ndev->mw_count)
  79		return -EINVAL;
  80
  81	return ndev->dev_data->mw_idx << idx;
  82}
  83
  84static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
  85{
  86	if (pidx != NTB_DEF_PEER_IDX)
  87		return -EINVAL;
  88
  89	return ntb_ndev(ntb)->mw_count;
  90}
  91
  92static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
  93				resource_size_t *addr_align,
  94				resource_size_t *size_align,
  95				resource_size_t *size_max)
  96{
  97	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  98	int bar;
  99
 100	if (pidx != NTB_DEF_PEER_IDX)
 101		return -EINVAL;
 102
 103	bar = ndev_mw_to_bar(ndev, idx);
 104	if (bar < 0)
 105		return bar;
 106
 107	if (addr_align)
 108		*addr_align = SZ_4K;
 109
 110	if (size_align)
 111		*size_align = 1;
 112
 113	if (size_max)
 114		*size_max = pci_resource_len(ndev->ntb.pdev, bar);
 115
 116	return 0;
 117}
 118
 119static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
 120				dma_addr_t addr, resource_size_t size)
 121{
 122	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 123	unsigned long xlat_reg, limit_reg = 0;
 124	resource_size_t mw_size;
 125	void __iomem *mmio, *peer_mmio;
 126	u64 base_addr, limit, reg_val;
 127	int bar;
 128
 129	if (pidx != NTB_DEF_PEER_IDX)
 130		return -EINVAL;
 131
 132	bar = ndev_mw_to_bar(ndev, idx);
 133	if (bar < 0)
 134		return bar;
 135
 136	mw_size = pci_resource_len(ntb->pdev, bar);
 137
 138	/* make sure the range fits in the usable mw size */
 139	if (size > mw_size)
 140		return -EINVAL;
 141
 142	mmio = ndev->self_mmio;
 143	peer_mmio = ndev->peer_mmio;
 144
 145	base_addr = pci_resource_start(ntb->pdev, bar);
 146
 147	if (bar != 1) {
 148		xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2);
 149		limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2);
 150
 151		/* Set the limit if supported */
 152		limit = size;
 153
 154		/* set and verify setting the translation address */
 155		write64(addr, peer_mmio + xlat_reg);
 156		reg_val = read64(peer_mmio + xlat_reg);
 157		if (reg_val != addr) {
 158			write64(0, peer_mmio + xlat_reg);
 159			return -EIO;
 160		}
 161
 162		/* set and verify setting the limit */
 163		write64(limit, peer_mmio + limit_reg);
 164		reg_val = read64(peer_mmio + limit_reg);
 165		if (reg_val != limit) {
 166			write64(base_addr, mmio + limit_reg);
 167			write64(0, peer_mmio + xlat_reg);
 168			return -EIO;
 169		}
 170	} else {
 171		xlat_reg = AMD_BAR1XLAT_OFFSET;
 172		limit_reg = AMD_BAR1LMT_OFFSET;
 173
 174		/* Set the limit if supported */
 175		limit = size;
 176
 177		/* set and verify setting the translation address */
 178		write64(addr, peer_mmio + xlat_reg);
 179		reg_val = read64(peer_mmio + xlat_reg);
 180		if (reg_val != addr) {
 181			write64(0, peer_mmio + xlat_reg);
 182			return -EIO;
 183		}
 184
 185		/* set and verify setting the limit */
 186		writel(limit, peer_mmio + limit_reg);
 187		reg_val = readl(peer_mmio + limit_reg);
 188		if (reg_val != limit) {
 189			writel(base_addr, mmio + limit_reg);
 190			writel(0, peer_mmio + xlat_reg);
 191			return -EIO;
 192		}
 193	}
 194
 195	return 0;
 196}
 197
 198static int amd_ntb_get_link_status(struct amd_ntb_dev *ndev)
 199{
 200	struct pci_dev *pdev = ndev->ntb.pdev;
 201	struct pci_dev *pci_swds = NULL;
 202	struct pci_dev *pci_swus = NULL;
 203	u32 stat;
 204	int rc;
 205
 206	if (ndev->ntb.topo == NTB_TOPO_SEC) {
 207		if (ndev->dev_data->is_endpoint) {
 208			rc = pcie_capability_read_dword(pdev, PCI_EXP_LNKCTL, &stat);
 209			if (rc)
 210				return rc;
 211
 212			ndev->lnk_sta = stat;
 213			return 0;
 214		}
 215
 216		/* Locate the pointer to Downstream Switch for this device */
 217		pci_swds = pci_upstream_bridge(ndev->ntb.pdev);
 218		if (pci_swds) {
 219			/*
 220			 * Locate the pointer to Upstream Switch for
 221			 * the Downstream Switch.
 222			 */
 223			pci_swus = pci_upstream_bridge(pci_swds);
 224			if (pci_swus) {
 225				rc = pcie_capability_read_dword(pci_swus,
 226								PCI_EXP_LNKCTL,
 227								&stat);
 228				if (rc)
 229					return 0;
 230			} else {
 231				return 0;
 232			}
 233		} else {
 234			return 0;
 235		}
 236	} else if (ndev->ntb.topo == NTB_TOPO_PRI) {
 237		/*
 238		 * For NTB primary, we simply read the Link Status and control
 239		 * register of the NTB device itself.
 240		 */
 241		pdev = ndev->ntb.pdev;
 242		rc = pcie_capability_read_dword(pdev, PCI_EXP_LNKCTL, &stat);
 243		if (rc)
 244			return 0;
 245	} else {
 246		/* Catch all for everything else */
 247		return 0;
 248	}
 249
 250	ndev->lnk_sta = stat;
 251
 252	return 1;
 253}
 254
 255static int amd_link_is_up(struct amd_ntb_dev *ndev)
 256{
 257	int ret;
 258
 259	/*
 260	 * We consider the link to be up under two conditions:
 261	 *
 262	 *   - When a link-up event is received. This is indicated by
 263	 *     AMD_LINK_UP_EVENT set in peer_sta.
 264	 *   - When driver on both sides of the link have been loaded.
 265	 *     This is indicated by bit 1 being set in the peer
 266	 *     SIDEINFO register.
 267	 *
 268	 * This function should return 1 when the latter of the above
 269	 * two conditions is true.
 270	 *
 271	 * Now consider the sequence of events - Link-Up event occurs,
 272	 * then the peer side driver loads. In this case, we would have
 273	 * received LINK_UP event and bit 1 of peer SIDEINFO is also
 274	 * set. What happens now if the link goes down? Bit 1 of
 275	 * peer SIDEINFO remains set, but LINK_DOWN bit is set in
 276	 * peer_sta. So we should return 0 from this function. Not only
 277	 * that, we clear bit 1 of peer SIDEINFO to 0, since the peer
 278	 * side driver did not even get a chance to clear it before
 279	 * the link went down. This can be the case of surprise link
 280	 * removal.
 281	 *
 282	 * LINK_UP event will always occur before the peer side driver
 283	 * gets loaded the very first time. So there can be a case when
 284	 * the LINK_UP event has occurred, but the peer side driver hasn't
 285	 * yet loaded. We return 0 in that case.
 286	 *
 287	 * There is also a special case when the primary side driver is
 288	 * unloaded and then loaded again. Since there is no change in
 289	 * the status of NTB secondary in this case, there is no Link-Up
 290	 * or Link-Down notification received. We recognize this condition
 291	 * with peer_sta being set to 0.
 292	 *
 293	 * If bit 1 of peer SIDEINFO register is not set, then we
 294	 * simply return 0 irrespective of the link up or down status
 295	 * set in peer_sta.
 296	 */
 297	ret = amd_poll_link(ndev);
 298	if (ret) {
 299		/*
 300		 * We need to check the below only for NTB primary. For NTB
 301		 * secondary, simply checking the result of PSIDE_INFO
 302		 * register will suffice.
 303		 */
 304		if (ndev->ntb.topo == NTB_TOPO_PRI) {
 305			if ((ndev->peer_sta & AMD_LINK_UP_EVENT) ||
 306			    (ndev->peer_sta == 0))
 307				return ret;
 308			else if (ndev->peer_sta & AMD_LINK_DOWN_EVENT) {
 309				/* Clear peer sideinfo register */
 310				amd_clear_side_info_reg(ndev, true);
 311
 312				return 0;
 313			}
 314		} else { /* NTB_TOPO_SEC */
 315			return ret;
 316		}
 317	}
 318
 319	return 0;
 320}
 321
 322static u64 amd_ntb_link_is_up(struct ntb_dev *ntb,
 323			      enum ntb_speed *speed,
 324			      enum ntb_width *width)
 325{
 326	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 327	int ret = 0;
 328
 329	if (amd_link_is_up(ndev)) {
 330		if (speed)
 331			*speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
 332		if (width)
 333			*width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
 334
 335		dev_dbg(&ntb->pdev->dev, "link is up.\n");
 336
 337		ret = 1;
 338	} else {
 339		if (speed)
 340			*speed = NTB_SPEED_NONE;
 341		if (width)
 342			*width = NTB_WIDTH_NONE;
 343
 344		dev_dbg(&ntb->pdev->dev, "link is down.\n");
 345	}
 346
 347	return ret;
 348}
 349
 350static int amd_ntb_link_enable(struct ntb_dev *ntb,
 351			       enum ntb_speed max_speed,
 352			       enum ntb_width max_width)
 353{
 354	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 355	void __iomem *mmio = ndev->self_mmio;
 356
 357	/* Enable event interrupt */
 358	ndev->int_mask &= ~AMD_EVENT_INTMASK;
 359	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
 360
 361	if (ndev->ntb.topo == NTB_TOPO_SEC)
 362		return -EINVAL;
 363	dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
 364
 365	return 0;
 366}
 367
 368static int amd_ntb_link_disable(struct ntb_dev *ntb)
 369{
 370	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 371	void __iomem *mmio = ndev->self_mmio;
 372
 373	/* Disable event interrupt */
 374	ndev->int_mask |= AMD_EVENT_INTMASK;
 375	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
 376
 377	if (ndev->ntb.topo == NTB_TOPO_SEC)
 378		return -EINVAL;
 379	dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
 380
 381	return 0;
 382}
 383
 384static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
 385{
 386	/* The same as for inbound MWs */
 387	return ntb_ndev(ntb)->mw_count;
 388}
 389
 390static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
 391				    phys_addr_t *base, resource_size_t *size)
 392{
 393	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 394	int bar;
 395
 396	bar = ndev_mw_to_bar(ndev, idx);
 397	if (bar < 0)
 398		return bar;
 399
 400	if (base)
 401		*base = pci_resource_start(ndev->ntb.pdev, bar);
 402
 403	if (size)
 404		*size = pci_resource_len(ndev->ntb.pdev, bar);
 405
 406	return 0;
 407}
 408
 409static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
 410{
 411	return ntb_ndev(ntb)->db_valid_mask;
 412}
 413
 414static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
 415{
 416	return ntb_ndev(ntb)->db_count;
 417}
 418
 419static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
 420{
 421	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 422
 423	if (db_vector < 0 || db_vector > ndev->db_count)
 424		return 0;
 425
 426	return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector);
 427}
 428
 429static u64 amd_ntb_db_read(struct ntb_dev *ntb)
 430{
 431	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 432	void __iomem *mmio = ndev->self_mmio;
 433
 434	return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
 435}
 436
 437static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
 438{
 439	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 440	void __iomem *mmio = ndev->self_mmio;
 441
 442	writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
 443
 444	return 0;
 445}
 446
 447static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
 448{
 449	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 450	void __iomem *mmio = ndev->self_mmio;
 451	unsigned long flags;
 452
 453	if (db_bits & ~ndev->db_valid_mask)
 454		return -EINVAL;
 455
 456	spin_lock_irqsave(&ndev->db_mask_lock, flags);
 457	ndev->db_mask |= db_bits;
 458	writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
 459	spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
 460
 461	return 0;
 462}
 463
 464static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
 465{
 466	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 467	void __iomem *mmio = ndev->self_mmio;
 468	unsigned long flags;
 469
 470	if (db_bits & ~ndev->db_valid_mask)
 471		return -EINVAL;
 472
 473	spin_lock_irqsave(&ndev->db_mask_lock, flags);
 474	ndev->db_mask &= ~db_bits;
 475	writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
 476	spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
 477
 478	return 0;
 479}
 480
 481static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
 482{
 483	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 484	void __iomem *mmio = ndev->self_mmio;
 485
 486	writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
 487
 488	return 0;
 489}
 490
 491static int amd_ntb_spad_count(struct ntb_dev *ntb)
 492{
 493	return ntb_ndev(ntb)->spad_count;
 494}
 495
 496static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
 497{
 498	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 499	void __iomem *mmio = ndev->self_mmio;
 500	u32 offset;
 501
 502	if (idx < 0 || idx >= ndev->spad_count)
 503		return 0;
 504
 505	offset = ndev->self_spad + (idx << 2);
 506	return readl(mmio + AMD_SPAD_OFFSET + offset);
 507}
 508
 509static int amd_ntb_spad_write(struct ntb_dev *ntb,
 510			      int idx, u32 val)
 511{
 512	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 513	void __iomem *mmio = ndev->self_mmio;
 514	u32 offset;
 515
 516	if (idx < 0 || idx >= ndev->spad_count)
 517		return -EINVAL;
 518
 519	offset = ndev->self_spad + (idx << 2);
 520	writel(val, mmio + AMD_SPAD_OFFSET + offset);
 521
 522	return 0;
 523}
 524
 525static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
 526{
 527	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 528	void __iomem *mmio = ndev->self_mmio;
 529	u32 offset;
 530
 531	if (sidx < 0 || sidx >= ndev->spad_count)
 532		return -EINVAL;
 533
 534	offset = ndev->peer_spad + (sidx << 2);
 535	return readl(mmio + AMD_SPAD_OFFSET + offset);
 536}
 537
 538static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
 539				   int sidx, u32 val)
 540{
 541	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 542	void __iomem *mmio = ndev->self_mmio;
 543	u32 offset;
 544
 545	if (sidx < 0 || sidx >= ndev->spad_count)
 546		return -EINVAL;
 547
 548	offset = ndev->peer_spad + (sidx << 2);
 549	writel(val, mmio + AMD_SPAD_OFFSET + offset);
 550
 551	return 0;
 552}
 553
 554static const struct ntb_dev_ops amd_ntb_ops = {
 555	.mw_count		= amd_ntb_mw_count,
 556	.mw_get_align		= amd_ntb_mw_get_align,
 557	.mw_set_trans		= amd_ntb_mw_set_trans,
 558	.peer_mw_count		= amd_ntb_peer_mw_count,
 559	.peer_mw_get_addr	= amd_ntb_peer_mw_get_addr,
 560	.link_is_up		= amd_ntb_link_is_up,
 561	.link_enable		= amd_ntb_link_enable,
 562	.link_disable		= amd_ntb_link_disable,
 563	.db_valid_mask		= amd_ntb_db_valid_mask,
 564	.db_vector_count	= amd_ntb_db_vector_count,
 565	.db_vector_mask		= amd_ntb_db_vector_mask,
 566	.db_read		= amd_ntb_db_read,
 567	.db_clear		= amd_ntb_db_clear,
 568	.db_set_mask		= amd_ntb_db_set_mask,
 569	.db_clear_mask		= amd_ntb_db_clear_mask,
 570	.peer_db_set		= amd_ntb_peer_db_set,
 571	.spad_count		= amd_ntb_spad_count,
 572	.spad_read		= amd_ntb_spad_read,
 573	.spad_write		= amd_ntb_spad_write,
 574	.peer_spad_read		= amd_ntb_peer_spad_read,
 575	.peer_spad_write	= amd_ntb_peer_spad_write,
 576};
 577
 578static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
 579{
 580	void __iomem *mmio = ndev->self_mmio;
 581	int reg;
 582
 583	reg = readl(mmio + AMD_SMUACK_OFFSET);
 584	reg |= bit;
 585	writel(reg, mmio + AMD_SMUACK_OFFSET);
 586}
 587
 588static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
 589{
 590	void __iomem *mmio = ndev->self_mmio;
 591	struct device *dev = &ndev->ntb.pdev->dev;
 592	u32 status;
 593
 594	status = readl(mmio + AMD_INTSTAT_OFFSET);
 595	if (!(status & AMD_EVENT_INTMASK))
 596		return;
 597
 598	dev_dbg(dev, "status = 0x%x and vec = %d\n", status, vec);
 599
 600	status &= AMD_EVENT_INTMASK;
 601	switch (status) {
 602	case AMD_PEER_FLUSH_EVENT:
 603		ndev->peer_sta |= AMD_PEER_FLUSH_EVENT;
 604		dev_info(dev, "Flush is done.\n");
 605		break;
 606	case AMD_PEER_RESET_EVENT:
 607	case AMD_LINK_DOWN_EVENT:
 608		ndev->peer_sta |= status;
 609		if (status == AMD_LINK_DOWN_EVENT)
 610			ndev->peer_sta &= ~AMD_LINK_UP_EVENT;
 611
 612		amd_ack_smu(ndev, status);
 613
 614		/* link down first */
 615		ntb_link_event(&ndev->ntb);
 616		/* polling peer status */
 617		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
 618
 619		break;
 620	case AMD_PEER_D3_EVENT:
 621	case AMD_PEER_PMETO_EVENT:
 622	case AMD_LINK_UP_EVENT:
 623		ndev->peer_sta |= status;
 624		if (status == AMD_LINK_UP_EVENT)
 625			ndev->peer_sta &= ~AMD_LINK_DOWN_EVENT;
 626		else if (status == AMD_PEER_D3_EVENT)
 627			ndev->peer_sta &= ~AMD_PEER_D0_EVENT;
 628
 629		amd_ack_smu(ndev, status);
 630
 631		/* link down */
 632		ntb_link_event(&ndev->ntb);
 633
 634		break;
 635	case AMD_PEER_D0_EVENT:
 636		mmio = ndev->peer_mmio;
 637		status = readl(mmio + AMD_PMESTAT_OFFSET);
 638		/* check if this is WAKEUP event */
 639		if (status & 0x1)
 640			dev_info(dev, "Wakeup is done.\n");
 641
 642		ndev->peer_sta |= AMD_PEER_D0_EVENT;
 643		ndev->peer_sta &= ~AMD_PEER_D3_EVENT;
 644		amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
 645
 646		/* start a timer to poll link status */
 647		schedule_delayed_work(&ndev->hb_timer,
 648				      AMD_LINK_HB_TIMEOUT);
 649		break;
 650	default:
 651		dev_info(dev, "event status = 0x%x.\n", status);
 652		break;
 653	}
 654
 655	/* Clear the interrupt status */
 656	writel(status, mmio + AMD_INTSTAT_OFFSET);
 657}
 658
 659static void amd_handle_db_event(struct amd_ntb_dev *ndev, int vec)
 660{
 661	struct device *dev = &ndev->ntb.pdev->dev;
 662	u64 status;
 663
 664	status = amd_ntb_db_read(&ndev->ntb);
 665
 666	dev_dbg(dev, "status = 0x%llx and vec = %d\n", status, vec);
 667
 668	/*
 669	 * Since we had reserved highest order bit of DB for signaling peer of
 670	 * a special event, this is the only status bit we should be concerned
 671	 * here now.
 672	 */
 673	if (status & BIT(ndev->db_last_bit)) {
 674		ntb_db_clear(&ndev->ntb, BIT(ndev->db_last_bit));
 675		/* send link down event notification */
 676		ntb_link_event(&ndev->ntb);
 677
 678		/*
 679		 * If we are here, that means the peer has signalled a special
 680		 * event which notifies that the peer driver has been
 681		 * un-loaded for some reason. Since there is a chance that the
 682		 * peer will load its driver again sometime, we schedule link
 683		 * polling routine.
 684		 */
 685		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
 686	}
 687}
 688
 689static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
 690{
 691	dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec);
 692
 693	if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
 694		amd_handle_event(ndev, vec);
 695
 696	if (vec < AMD_DB_CNT) {
 697		amd_handle_db_event(ndev, vec);
 698		ntb_db_event(&ndev->ntb, vec);
 699	}
 700
 701	return IRQ_HANDLED;
 702}
 703
 704static irqreturn_t ndev_vec_isr(int irq, void *dev)
 705{
 706	struct amd_ntb_vec *nvec = dev;
 707
 708	return ndev_interrupt(nvec->ndev, nvec->num);
 709}
 710
 711static irqreturn_t ndev_irq_isr(int irq, void *dev)
 712{
 713	struct amd_ntb_dev *ndev = dev;
 714
 715	return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
 716}
 717
 718static int ndev_init_isr(struct amd_ntb_dev *ndev,
 719			 int msix_min, int msix_max)
 720{
 721	struct pci_dev *pdev;
 722	int rc, i, msix_count, node;
 723
 724	pdev = ndev->ntb.pdev;
 725
 726	node = dev_to_node(&pdev->dev);
 727
 728	ndev->db_mask = ndev->db_valid_mask;
 729
 730	/* Try to set up msix irq */
 731	ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
 732				 GFP_KERNEL, node);
 733	if (!ndev->vec)
 734		goto err_msix_vec_alloc;
 735
 736	ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
 737				  GFP_KERNEL, node);
 738	if (!ndev->msix)
 739		goto err_msix_alloc;
 740
 741	for (i = 0; i < msix_max; ++i)
 742		ndev->msix[i].entry = i;
 743
 744	msix_count = pci_enable_msix_range(pdev, ndev->msix,
 745					   msix_min, msix_max);
 746	if (msix_count < 0)
 747		goto err_msix_enable;
 748
 749	/* NOTE: Disable MSIX if msix count is less than 16 because of
 750	 * hardware limitation.
 751	 */
 752	if (msix_count < msix_min) {
 753		pci_disable_msix(pdev);
 754		goto err_msix_enable;
 755	}
 756
 757	for (i = 0; i < msix_count; ++i) {
 758		ndev->vec[i].ndev = ndev;
 759		ndev->vec[i].num = i;
 760		rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
 761				 "ndev_vec_isr", &ndev->vec[i]);
 762		if (rc)
 763			goto err_msix_request;
 764	}
 765
 766	dev_dbg(&pdev->dev, "Using msix interrupts\n");
 767	ndev->db_count = msix_min;
 768	ndev->msix_vec_count = msix_max;
 769	return 0;
 770
 771err_msix_request:
 772	while (i-- > 0)
 773		free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 774	pci_disable_msix(pdev);
 775err_msix_enable:
 776	kfree(ndev->msix);
 777err_msix_alloc:
 778	kfree(ndev->vec);
 779err_msix_vec_alloc:
 780	ndev->msix = NULL;
 781	ndev->vec = NULL;
 782
 783	/* Try to set up msi irq */
 784	rc = pci_enable_msi(pdev);
 785	if (rc)
 786		goto err_msi_enable;
 787
 788	rc = request_irq(pdev->irq, ndev_irq_isr, 0,
 789			 "ndev_irq_isr", ndev);
 790	if (rc)
 791		goto err_msi_request;
 792
 793	dev_dbg(&pdev->dev, "Using msi interrupts\n");
 794	ndev->db_count = 1;
 795	ndev->msix_vec_count = 1;
 796	return 0;
 797
 798err_msi_request:
 799	pci_disable_msi(pdev);
 800err_msi_enable:
 801
 802	/* Try to set up intx irq */
 803	pci_intx(pdev, 1);
 804
 805	rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
 806			 "ndev_irq_isr", ndev);
 807	if (rc)
 808		goto err_intx_request;
 809
 810	dev_dbg(&pdev->dev, "Using intx interrupts\n");
 811	ndev->db_count = 1;
 812	ndev->msix_vec_count = 1;
 813	return 0;
 814
 815err_intx_request:
 816	return rc;
 817}
 818
 819static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
 820{
 821	struct pci_dev *pdev;
 822	void __iomem *mmio = ndev->self_mmio;
 823	int i;
 824
 825	pdev = ndev->ntb.pdev;
 826
 827	/* Mask all doorbell interrupts */
 828	ndev->db_mask = ndev->db_valid_mask;
 829	writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
 830
 831	if (ndev->msix) {
 832		i = ndev->msix_vec_count;
 833		while (i--)
 834			free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 835		pci_disable_msix(pdev);
 836		kfree(ndev->msix);
 837		kfree(ndev->vec);
 838	} else {
 839		free_irq(pdev->irq, ndev);
 840		if (pci_dev_msi_enabled(pdev))
 841			pci_disable_msi(pdev);
 842		else
 843			pci_intx(pdev, 0);
 844	}
 845}
 846
 847static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
 848				 size_t count, loff_t *offp)
 849{
 850	struct amd_ntb_dev *ndev;
 851	void __iomem *mmio;
 852	char *buf;
 853	size_t buf_size;
 854	ssize_t ret, off;
 855	union { u64 v64; u32 v32; u16 v16; } u;
 856
 857	ndev = filp->private_data;
 858	mmio = ndev->self_mmio;
 859
 860	buf_size = min(count, 0x800ul);
 861
 862	buf = kmalloc(buf_size, GFP_KERNEL);
 863	if (!buf)
 864		return -ENOMEM;
 865
 866	off = 0;
 867
 868	off += scnprintf(buf + off, buf_size - off,
 869			 "NTB Device Information:\n");
 870
 871	off += scnprintf(buf + off, buf_size - off,
 872			 "Connection Topology -\t%s\n",
 873			 ntb_topo_string(ndev->ntb.topo));
 874
 875	off += scnprintf(buf + off, buf_size - off,
 876			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
 877
 878	if (!amd_link_is_up(ndev)) {
 879		off += scnprintf(buf + off, buf_size - off,
 880				 "Link Status -\t\tDown\n");
 881	} else {
 882		off += scnprintf(buf + off, buf_size - off,
 883				 "Link Status -\t\tUp\n");
 884		off += scnprintf(buf + off, buf_size - off,
 885				 "Link Speed -\t\tPCI-E Gen %u\n",
 886				 NTB_LNK_STA_SPEED(ndev->lnk_sta));
 887		off += scnprintf(buf + off, buf_size - off,
 888				 "Link Width -\t\tx%u\n",
 889				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
 890	}
 891
 892	off += scnprintf(buf + off, buf_size - off,
 893			 "Memory Window Count -\t%u\n", ndev->mw_count);
 894	off += scnprintf(buf + off, buf_size - off,
 895			 "Scratchpad Count -\t%u\n", ndev->spad_count);
 896	off += scnprintf(buf + off, buf_size - off,
 897			 "Doorbell Count -\t%u\n", ndev->db_count);
 898	off += scnprintf(buf + off, buf_size - off,
 899			 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
 900
 901	off += scnprintf(buf + off, buf_size - off,
 902			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
 903
 904	u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
 905	off += scnprintf(buf + off, buf_size - off,
 906			 "Doorbell Mask -\t\t\t%#06x\n", u.v32);
 907
 908	u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
 909	off += scnprintf(buf + off, buf_size - off,
 910			 "Doorbell Bell -\t\t\t%#06x\n", u.v32);
 911
 912	off += scnprintf(buf + off, buf_size - off,
 913			 "\nNTB Incoming XLAT:\n");
 914
 915	u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
 916	off += scnprintf(buf + off, buf_size - off,
 917			 "XLAT1 -\t\t%#018llx\n", u.v64);
 918
 919	u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
 920	off += scnprintf(buf + off, buf_size - off,
 921			 "XLAT23 -\t\t%#018llx\n", u.v64);
 922
 923	u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
 924	off += scnprintf(buf + off, buf_size - off,
 925			 "XLAT45 -\t\t%#018llx\n", u.v64);
 926
 927	u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
 928	off += scnprintf(buf + off, buf_size - off,
 929			 "LMT1 -\t\t\t%#06x\n", u.v32);
 930
 931	u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
 932	off += scnprintf(buf + off, buf_size - off,
 933			 "LMT23 -\t\t\t%#018llx\n", u.v64);
 934
 935	u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
 936	off += scnprintf(buf + off, buf_size - off,
 937			 "LMT45 -\t\t\t%#018llx\n", u.v64);
 938
 939	ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
 940	kfree(buf);
 941	return ret;
 942}
 943
 944static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
 945{
 946	if (!debugfs_dir) {
 947		ndev->debugfs_dir = NULL;
 948		ndev->debugfs_info = NULL;
 949	} else {
 950		ndev->debugfs_dir =
 951			debugfs_create_dir(pci_name(ndev->ntb.pdev),
 952					   debugfs_dir);
 953		ndev->debugfs_info =
 954			debugfs_create_file("info", S_IRUSR,
 955					    ndev->debugfs_dir, ndev,
 956					    &amd_ntb_debugfs_info);
 957	}
 958}
 959
 960static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
 961{
 962	debugfs_remove_recursive(ndev->debugfs_dir);
 963}
 964
 965static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
 966				    struct pci_dev *pdev)
 967{
 968	ndev->ntb.pdev = pdev;
 969	ndev->ntb.topo = NTB_TOPO_NONE;
 970	ndev->ntb.ops = &amd_ntb_ops;
 971	ndev->int_mask = AMD_EVENT_INTMASK;
 972	spin_lock_init(&ndev->db_mask_lock);
 973}
 974
 975static int amd_poll_link(struct amd_ntb_dev *ndev)
 976{
 977	void __iomem *mmio = ndev->peer_mmio;
 978	u32 reg;
 979
 980	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
 981	reg &= AMD_SIDE_READY;
 982
 983	dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg);
 984
 985	ndev->cntl_sta = reg;
 986
 987	amd_ntb_get_link_status(ndev);
 988
 989	return ndev->cntl_sta;
 990}
 991
 992static void amd_link_hb(struct work_struct *work)
 993{
 994	struct amd_ntb_dev *ndev = hb_ndev(work);
 995
 996	if (amd_poll_link(ndev))
 997		ntb_link_event(&ndev->ntb);
 998
 999	if (!amd_link_is_up(ndev))
1000		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
1001}
1002
1003static int amd_init_isr(struct amd_ntb_dev *ndev)
1004{
1005	return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
1006}
1007
1008static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer)
1009{
1010	void __iomem *mmio = NULL;
1011	unsigned int reg;
1012
1013	if (peer)
1014		mmio = ndev->peer_mmio;
1015	else
1016		mmio = ndev->self_mmio;
1017
1018	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
1019	if (!(reg & AMD_SIDE_READY)) {
1020		reg |= AMD_SIDE_READY;
1021		writel(reg, mmio + AMD_SIDEINFO_OFFSET);
1022	}
1023}
1024
1025static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer)
1026{
1027	void __iomem *mmio = NULL;
1028	unsigned int reg;
1029
1030	if (peer)
1031		mmio = ndev->peer_mmio;
1032	else
1033		mmio = ndev->self_mmio;
1034
1035	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
1036	if (reg & AMD_SIDE_READY) {
1037		reg &= ~AMD_SIDE_READY;
1038		writel(reg, mmio + AMD_SIDEINFO_OFFSET);
1039		readl(mmio + AMD_SIDEINFO_OFFSET);
1040	}
1041}
1042
1043static void amd_init_side_info(struct amd_ntb_dev *ndev)
1044{
1045	void __iomem *mmio = ndev->self_mmio;
1046	u32 ntb_ctl;
1047
1048	amd_set_side_info_reg(ndev, false);
1049
1050	ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
1051	ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
1052	writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
1053}
1054
1055static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
1056{
1057	void __iomem *mmio = ndev->self_mmio;
1058	u32 ntb_ctl;
1059
1060	amd_clear_side_info_reg(ndev, false);
1061
1062	ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
1063	ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
1064	writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
1065}
1066
1067static int amd_init_ntb(struct amd_ntb_dev *ndev)
1068{
1069	void __iomem *mmio = ndev->self_mmio;
1070
1071	ndev->mw_count = ndev->dev_data->mw_count;
1072	ndev->spad_count = AMD_SPADS_CNT;
1073	ndev->db_count = AMD_DB_CNT;
1074
1075	switch (ndev->ntb.topo) {
1076	case NTB_TOPO_PRI:
1077	case NTB_TOPO_SEC:
1078		ndev->spad_count >>= 1;
1079		if (ndev->ntb.topo == NTB_TOPO_PRI) {
1080			ndev->self_spad = 0;
1081			ndev->peer_spad = 0x20;
1082		} else {
1083			ndev->self_spad = 0x20;
1084			ndev->peer_spad = 0;
1085		}
1086
1087		INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
1088		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
1089
1090		break;
1091	default:
1092		dev_err(&ndev->ntb.pdev->dev,
1093			"AMD NTB does not support B2B mode.\n");
1094		return -EINVAL;
1095	}
1096
1097	/* Mask event interrupts */
1098	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
1099
1100	return 0;
1101}
1102
1103static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
1104{
1105	void __iomem *mmio = ndev->self_mmio;
1106	u32 info;
1107
1108	info = readl(mmio + AMD_SIDEINFO_OFFSET);
1109	if (info & AMD_SIDE_MASK)
1110		return NTB_TOPO_SEC;
1111	else
1112		return NTB_TOPO_PRI;
1113}
1114
1115static int amd_init_dev(struct amd_ntb_dev *ndev)
1116{
1117	void __iomem *mmio = ndev->self_mmio;
1118	struct pci_dev *pdev;
1119	int rc = 0;
1120
1121	pdev = ndev->ntb.pdev;
1122
1123	ndev->ntb.topo = amd_get_topo(ndev);
1124	dev_dbg(&pdev->dev, "AMD NTB topo is %s\n",
1125		ntb_topo_string(ndev->ntb.topo));
1126
1127	rc = amd_init_ntb(ndev);
1128	if (rc)
1129		return rc;
1130
1131	rc = amd_init_isr(ndev);
1132	if (rc) {
1133		dev_err(&pdev->dev, "fail to init isr.\n");
1134		return rc;
1135	}
1136
1137	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1138	/*
1139	 * We reserve the highest order bit of the DB register which will
1140	 * be used to notify peer when the driver on this side is being
1141	 * un-loaded.
1142	 */
1143	ndev->db_last_bit =
1144			find_last_bit((unsigned long *)&ndev->db_valid_mask,
1145				      hweight64(ndev->db_valid_mask));
1146	writew((u16)~BIT(ndev->db_last_bit), mmio + AMD_DBMASK_OFFSET);
1147	/*
1148	 * Since now there is one less bit to account for, the DB count
1149	 * and DB mask should be adjusted accordingly.
1150	 */
1151	ndev->db_count -= 1;
1152	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1153
1154	/* Enable Link-Up and Link-Down event interrupts */
1155	ndev->int_mask &= ~(AMD_LINK_UP_EVENT | AMD_LINK_DOWN_EVENT);
1156	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
1157
1158	return 0;
1159}
1160
1161static void amd_deinit_dev(struct amd_ntb_dev *ndev)
1162{
1163	cancel_delayed_work_sync(&ndev->hb_timer);
1164
1165	ndev_deinit_isr(ndev);
1166}
1167
1168static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
1169			    struct pci_dev *pdev)
1170{
1171	int rc;
1172
1173	pci_set_drvdata(pdev, ndev);
1174
1175	rc = pci_enable_device(pdev);
1176	if (rc)
1177		goto err_pci_enable;
1178
1179	rc = pci_request_regions(pdev, NTB_NAME);
1180	if (rc)
1181		goto err_pci_regions;
1182
1183	pci_set_master(pdev);
1184
1185	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1186	if (rc) {
1187		rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1188		if (rc)
1189			goto err_dma_mask;
1190		dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1191	}
1192
1193	ndev->self_mmio = pci_iomap(pdev, 0, 0);
1194	if (!ndev->self_mmio) {
1195		rc = -EIO;
1196		goto err_dma_mask;
1197	}
1198	ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
1199
1200	return 0;
1201
1202err_dma_mask:
1203	pci_release_regions(pdev);
1204err_pci_regions:
1205	pci_disable_device(pdev);
1206err_pci_enable:
1207	pci_set_drvdata(pdev, NULL);
1208	return rc;
1209}
1210
1211static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
1212{
1213	struct pci_dev *pdev = ndev->ntb.pdev;
1214
1215	pci_iounmap(pdev, ndev->self_mmio);
1216
1217	pci_release_regions(pdev);
1218	pci_disable_device(pdev);
1219	pci_set_drvdata(pdev, NULL);
1220}
1221
1222static int amd_ntb_pci_probe(struct pci_dev *pdev,
1223			     const struct pci_device_id *id)
1224{
1225	struct amd_ntb_dev *ndev;
1226	int rc, node;
1227
1228	node = dev_to_node(&pdev->dev);
1229
1230	ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1231	if (!ndev) {
1232		rc = -ENOMEM;
1233		goto err_ndev;
1234	}
1235
1236	ndev->dev_data = (struct ntb_dev_data *)id->driver_data;
1237
1238	ndev_init_struct(ndev, pdev);
1239
1240	rc = amd_ntb_init_pci(ndev, pdev);
1241	if (rc)
1242		goto err_init_pci;
1243
1244	rc = amd_init_dev(ndev);
1245	if (rc)
1246		goto err_init_dev;
1247
1248	/* write side info */
1249	amd_init_side_info(ndev);
1250
1251	amd_poll_link(ndev);
1252
1253	ndev_init_debugfs(ndev);
1254
1255	rc = ntb_register_device(&ndev->ntb);
1256	if (rc)
1257		goto err_register;
1258
1259	dev_info(&pdev->dev, "NTB device registered.\n");
1260
1261	return 0;
1262
1263err_register:
1264	ndev_deinit_debugfs(ndev);
1265	amd_deinit_dev(ndev);
1266err_init_dev:
1267	amd_ntb_deinit_pci(ndev);
1268err_init_pci:
1269	kfree(ndev);
1270err_ndev:
1271	return rc;
1272}
1273
1274static void amd_ntb_pci_remove(struct pci_dev *pdev)
1275{
1276	struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1277
1278	/*
1279	 * Clear the READY bit in SIDEINFO register before sending DB event
1280	 * to the peer. This will make sure that when the peer handles the
1281	 * DB event, it correctly reads this bit as being 0.
1282	 */
1283	amd_deinit_side_info(ndev);
1284	ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit));
1285	ntb_unregister_device(&ndev->ntb);
1286	ndev_deinit_debugfs(ndev);
1287	amd_deinit_dev(ndev);
1288	amd_ntb_deinit_pci(ndev);
1289	kfree(ndev);
1290}
1291
1292static void amd_ntb_pci_shutdown(struct pci_dev *pdev)
1293{
1294	struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1295
1296	/* Send link down notification */
1297	ntb_link_event(&ndev->ntb);
1298
1299	amd_deinit_side_info(ndev);
1300	ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit));
1301	ntb_unregister_device(&ndev->ntb);
1302	ndev_deinit_debugfs(ndev);
1303	amd_deinit_dev(ndev);
1304	amd_ntb_deinit_pci(ndev);
1305	kfree(ndev);
1306}
1307
1308static const struct file_operations amd_ntb_debugfs_info = {
1309	.owner = THIS_MODULE,
1310	.open = simple_open,
1311	.read = ndev_debugfs_read,
1312};
1313
1314static const struct ntb_dev_data dev_data[] = {
1315	{ /* for device 145b */
1316		.mw_count = 3,
1317		.mw_idx = 1,
1318	},
1319	{ /* for device 148b */
1320		.mw_count = 2,
1321		.mw_idx = 2,
1322	},
1323	{ /* for device 0x17d7 */
1324		.mw_count = 2,
1325		.mw_idx = 2,
1326		.is_endpoint = true,
1327	},
1328};
1329
1330static const struct pci_device_id amd_ntb_pci_tbl[] = {
1331	{ PCI_VDEVICE(AMD, 0x145b), (kernel_ulong_t)&dev_data[0] },
1332	{ PCI_VDEVICE(AMD, 0x148b), (kernel_ulong_t)&dev_data[1] },
1333	{ PCI_VDEVICE(AMD, 0x14c0), (kernel_ulong_t)&dev_data[1] },
1334	{ PCI_VDEVICE(AMD, 0x14c3), (kernel_ulong_t)&dev_data[1] },
1335	{ PCI_VDEVICE(AMD, 0x155a), (kernel_ulong_t)&dev_data[1] },
1336	{ PCI_VDEVICE(AMD, 0x17d4), (kernel_ulong_t)&dev_data[1] },
1337	{ PCI_VDEVICE(AMD, 0x17d7), (kernel_ulong_t)&dev_data[2] },
1338	{ PCI_VDEVICE(HYGON, 0x145b), (kernel_ulong_t)&dev_data[0] },
1339	{ 0, }
1340};
1341MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
1342
1343static struct pci_driver amd_ntb_pci_driver = {
1344	.name		= KBUILD_MODNAME,
1345	.id_table	= amd_ntb_pci_tbl,
1346	.probe		= amd_ntb_pci_probe,
1347	.remove		= amd_ntb_pci_remove,
1348	.shutdown	= amd_ntb_pci_shutdown,
1349};
1350
1351static int __init amd_ntb_pci_driver_init(void)
1352{
1353	int ret;
1354	pr_info("%s %s\n", NTB_DESC, NTB_VER);
1355
1356	if (debugfs_initialized())
1357		debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1358
1359	ret = pci_register_driver(&amd_ntb_pci_driver);
1360	if (ret)
1361		debugfs_remove_recursive(debugfs_dir);
1362
1363	return ret;
1364}
1365module_init(amd_ntb_pci_driver_init);
1366
1367static void __exit amd_ntb_pci_driver_exit(void)
1368{
1369	pci_unregister_driver(&amd_ntb_pci_driver);
1370	debugfs_remove_recursive(debugfs_dir);
1371}
1372module_exit(amd_ntb_pci_driver_exit);