icm.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Internal Thunderbolt Connection Manager. This is a firmware running on
   4 * the Thunderbolt host controller performing most of the low-level
   5 * handling.
   6 *
   7 * Copyright (C) 2017, Intel Corporation
   8 * Authors: Michael Jamet <michael.jamet@intel.com>
   9 *          Mika Westerberg <mika.westerberg@linux.intel.com>
  10 */
  11
  12#include <linux/delay.h>
  13#include <linux/mutex.h>
  14#include <linux/moduleparam.h>
  15#include <linux/pci.h>
  16#include <linux/pm_runtime.h>
  17#include <linux/platform_data/x86/apple.h>
  18#include <linux/sizes.h>
  19#include <linux/slab.h>
  20#include <linux/workqueue.h>
  21
  22#include "ctl.h"
  23#include "nhi_regs.h"
  24#include "tb.h"
  25#include "tunnel.h"
  26
  27#define PCIE2CIO_CMD			0x30
  28#define PCIE2CIO_CMD_TIMEOUT		BIT(31)
  29#define PCIE2CIO_CMD_START		BIT(30)
  30#define PCIE2CIO_CMD_WRITE		BIT(21)
  31#define PCIE2CIO_CMD_CS_MASK		GENMASK(20, 19)
  32#define PCIE2CIO_CMD_CS_SHIFT		19
  33#define PCIE2CIO_CMD_PORT_MASK		GENMASK(18, 13)
  34#define PCIE2CIO_CMD_PORT_SHIFT		13
  35
  36#define PCIE2CIO_WRDATA			0x34
  37#define PCIE2CIO_RDDATA			0x38
  38
  39#define PHY_PORT_CS1			0x37
  40#define PHY_PORT_CS1_LINK_DISABLE	BIT(14)
  41#define PHY_PORT_CS1_LINK_STATE_MASK	GENMASK(29, 26)
  42#define PHY_PORT_CS1_LINK_STATE_SHIFT	26
  43
  44#define ICM_TIMEOUT			5000	/* ms */
  45#define ICM_RETRIES			3
  46#define ICM_APPROVE_TIMEOUT		10000	/* ms */
  47#define ICM_MAX_LINK			4
  48
  49static bool start_icm;
  50module_param(start_icm, bool, 0444);
  51MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");
  52
  53/**
  54 * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status
  55 * @reply: Reply from ICM firmware is placed here
  56 * @request: Request that is sent to ICM firmware
  57 * @icm: Pointer to ICM private data
  58 */
  59struct usb4_switch_nvm_auth {
  60	struct icm_usb4_switch_op_response reply;
  61	struct icm_usb4_switch_op request;
  62	struct icm *icm;
  63};
  64
  65/**
  66 * struct icm - Internal connection manager private data
  67 * @request_lock: Makes sure only one message is send to ICM at time
  68 * @rescan_work: Work used to rescan the surviving switches after resume
  69 * @upstream_port: Pointer to the PCIe upstream port this host
  70 *		   controller is connected. This is only set for systems
  71 *		   where ICM needs to be started manually
  72 * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides
  73 *	     (only set when @upstream_port is not %NULL)
  74 * @safe_mode: ICM is in safe mode
  75 * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)
  76 * @rpm: Does the controller support runtime PM (RTD3)
  77 * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller
  78 * @proto_version: Firmware protocol version
  79 * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)
  80 * @veto: Is RTD3 veto in effect
  81 * @is_supported: Checks if we can support ICM on this controller
  82 * @cio_reset: Trigger CIO reset
  83 * @get_mode: Read and return the ICM firmware mode (optional)
  84 * @get_route: Find a route string for given switch
  85 * @save_devices: Ask ICM to save devices to ACL when suspending (optional)
  86 * @driver_ready: Send driver ready message to ICM
  87 * @set_uuid: Set UUID for the root switch (optional)
  88 * @device_connected: Handle device connected ICM message
  89 * @device_disconnected: Handle device disconnected ICM message
  90 * @xdomain_connected: Handle XDomain connected ICM message
  91 * @xdomain_disconnected: Handle XDomain disconnected ICM message
  92 * @rtd3_veto: Handle RTD3 veto notification ICM message
  93 */
  94struct icm {
  95	struct mutex request_lock;
  96	struct delayed_work rescan_work;
  97	struct pci_dev *upstream_port;
  98	int vnd_cap;
  99	bool safe_mode;
 100	size_t max_boot_acl;
 101	bool rpm;
 102	bool can_upgrade_nvm;
 103	u8 proto_version;
 104	struct usb4_switch_nvm_auth *last_nvm_auth;
 105	bool veto;
 106	bool (*is_supported)(struct tb *tb);
 107	int (*cio_reset)(struct tb *tb);
 108	int (*get_mode)(struct tb *tb);
 109	int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);
 110	void (*save_devices)(struct tb *tb);
 111	int (*driver_ready)(struct tb *tb,
 112			    enum tb_security_level *security_level,
 113			    u8 *proto_version, size_t *nboot_acl, bool *rpm);
 114	void (*set_uuid)(struct tb *tb);
 115	void (*device_connected)(struct tb *tb,
 116				 const struct icm_pkg_header *hdr);
 117	void (*device_disconnected)(struct tb *tb,
 118				    const struct icm_pkg_header *hdr);
 119	void (*xdomain_connected)(struct tb *tb,
 120				  const struct icm_pkg_header *hdr);
 121	void (*xdomain_disconnected)(struct tb *tb,
 122				     const struct icm_pkg_header *hdr);
 123	void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);
 124};
 125
 126struct icm_notification {
 127	struct work_struct work;
 128	struct icm_pkg_header *pkg;
 129	struct tb *tb;
 130};
 131
 132struct ep_name_entry {
 133	u8 len;
 134	u8 type;
 135	u8 data[];
 136};
 137
 138#define EP_NAME_INTEL_VSS	0x10
 139
 140/* Intel Vendor specific structure */
 141struct intel_vss {
 142	u16 vendor;
 143	u16 model;
 144	u8 mc;
 145	u8 flags;
 146	u16 pci_devid;
 147	u32 nvm_version;
 148};
 149
 150#define INTEL_VSS_FLAGS_RTD3	BIT(0)
 151
 152static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)
 153{
 154	const void *end = ep_name + size;
 155
 156	while (ep_name < end) {
 157		const struct ep_name_entry *ep = ep_name;
 158
 159		if (!ep->len)
 160			break;
 161		if (ep_name + ep->len > end)
 162			break;
 163
 164		if (ep->type == EP_NAME_INTEL_VSS)
 165			return (const struct intel_vss *)ep->data;
 166
 167		ep_name += ep->len;
 168	}
 169
 170	return NULL;
 171}
 172
 173static bool intel_vss_is_rtd3(const void *ep_name, size_t size)
 174{
 175	const struct intel_vss *vss;
 176
 177	vss = parse_intel_vss(ep_name, size);
 178	if (vss)
 179		return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);
 180
 181	return false;
 182}
 183
 184static inline struct tb *icm_to_tb(struct icm *icm)
 185{
 186	return ((void *)icm - sizeof(struct tb));
 187}
 188
 189static inline u8 phy_port_from_route(u64 route, u8 depth)
 190{
 191	u8 link;
 192
 193	link = depth ? route >> ((depth - 1) * 8) : route;
 194	return tb_phy_port_from_link(link);
 195}
 196
 197static inline u8 dual_link_from_link(u8 link)
 198{
 199	return link ? ((link - 1) ^ 0x01) + 1 : 0;
 200}
 201
 202static inline u64 get_route(u32 route_hi, u32 route_lo)
 203{
 204	return (u64)route_hi << 32 | route_lo;
 205}
 206
 207static inline u64 get_parent_route(u64 route)
 208{
 209	int depth = tb_route_length(route);
 210	return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;
 211}
 212
 213static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)
 214{
 215	unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);
 216	u32 cmd;
 217
 218	do {
 219		pci_read_config_dword(icm->upstream_port,
 220				      icm->vnd_cap + PCIE2CIO_CMD, &cmd);
 221		if (!(cmd & PCIE2CIO_CMD_START)) {
 222			if (cmd & PCIE2CIO_CMD_TIMEOUT)
 223				break;
 224			return 0;
 225		}
 226
 227		msleep(50);
 228	} while (time_before(jiffies, end));
 229
 230	return -ETIMEDOUT;
 231}
 232
 233static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,
 234			 unsigned int port, unsigned int index, u32 *data)
 235{
 236	struct pci_dev *pdev = icm->upstream_port;
 237	int ret, vnd_cap = icm->vnd_cap;
 238	u32 cmd;
 239
 240	cmd = index;
 241	cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
 242	cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
 243	cmd |= PCIE2CIO_CMD_START;
 244	pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
 245
 246	ret = pci2cio_wait_completion(icm, 5000);
 247	if (ret)
 248		return ret;
 249
 250	pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);
 251	return 0;
 252}
 253
 254static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,
 255			  unsigned int port, unsigned int index, u32 data)
 256{
 257	struct pci_dev *pdev = icm->upstream_port;
 258	int vnd_cap = icm->vnd_cap;
 259	u32 cmd;
 260
 261	pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);
 262
 263	cmd = index;
 264	cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
 265	cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
 266	cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;
 267	pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
 268
 269	return pci2cio_wait_completion(icm, 5000);
 270}
 271
 272static bool icm_match(const struct tb_cfg_request *req,
 273		      const struct ctl_pkg *pkg)
 274{
 275	const struct icm_pkg_header *res_hdr = pkg->buffer;
 276	const struct icm_pkg_header *req_hdr = req->request;
 277
 278	if (pkg->frame.eof != req->response_type)
 279		return false;
 280	if (res_hdr->code != req_hdr->code)
 281		return false;
 282
 283	return true;
 284}
 285
 286static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)
 287{
 288	const struct icm_pkg_header *hdr = pkg->buffer;
 289
 290	if (hdr->packet_id < req->npackets) {
 291		size_t offset = hdr->packet_id * req->response_size;
 292
 293		memcpy(req->response + offset, pkg->buffer, req->response_size);
 294	}
 295
 296	return hdr->packet_id == hdr->total_packets - 1;
 297}
 298
 299static int icm_request(struct tb *tb, const void *request, size_t request_size,
 300		       void *response, size_t response_size, size_t npackets,
 301		       int retries, unsigned int timeout_msec)
 302{
 303	struct icm *icm = tb_priv(tb);
 304
 305	do {
 306		struct tb_cfg_request *req;
 307		struct tb_cfg_result res;
 308
 309		req = tb_cfg_request_alloc();
 310		if (!req)
 311			return -ENOMEM;
 312
 313		req->match = icm_match;
 314		req->copy = icm_copy;
 315		req->request = request;
 316		req->request_size = request_size;
 317		req->request_type = TB_CFG_PKG_ICM_CMD;
 318		req->response = response;
 319		req->npackets = npackets;
 320		req->response_size = response_size;
 321		req->response_type = TB_CFG_PKG_ICM_RESP;
 322
 323		mutex_lock(&icm->request_lock);
 324		res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);
 325		mutex_unlock(&icm->request_lock);
 326
 327		tb_cfg_request_put(req);
 328
 329		if (res.err != -ETIMEDOUT)
 330			return res.err == 1 ? -EIO : res.err;
 331
 332		usleep_range(20, 50);
 333	} while (retries--);
 334
 335	return -ETIMEDOUT;
 336}
 337
 338/*
 339 * If rescan is queued to run (we are resuming), postpone it to give the
 340 * firmware some more time to send device connected notifications for next
 341 * devices in the chain.
 342 */
 343static void icm_postpone_rescan(struct tb *tb)
 344{
 345	struct icm *icm = tb_priv(tb);
 346
 347	if (delayed_work_pending(&icm->rescan_work))
 348		mod_delayed_work(tb->wq, &icm->rescan_work,
 349				 msecs_to_jiffies(500));
 350}
 351
 352static void icm_veto_begin(struct tb *tb)
 353{
 354	struct icm *icm = tb_priv(tb);
 355
 356	if (!icm->veto) {
 357		icm->veto = true;
 358		/* Keep the domain powered while veto is in effect */
 359		pm_runtime_get(&tb->dev);
 360	}
 361}
 362
 363static void icm_veto_end(struct tb *tb)
 364{
 365	struct icm *icm = tb_priv(tb);
 366
 367	if (icm->veto) {
 368		icm->veto = false;
 369		/* Allow the domain suspend now */
 370		pm_runtime_mark_last_busy(&tb->dev);
 371		pm_runtime_put_autosuspend(&tb->dev);
 372	}
 373}
 374
 375static bool icm_firmware_running(const struct tb_nhi *nhi)
 376{
 377	u32 val;
 378
 379	val = ioread32(nhi->iobase + REG_FW_STS);
 380	return !!(val & REG_FW_STS_ICM_EN);
 381}
 382
 383static void icm_xdomain_activated(struct tb_xdomain *xd, bool activated)
 384{
 385	struct tb_port *nhi_port, *dst_port;
 386	struct tb *tb = xd->tb;
 387
 388	nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
 389	dst_port = tb_xdomain_downstream_port(xd);
 390
 391	if (activated)
 392		tb_tunnel_event(tb, TB_TUNNEL_ACTIVATED, TB_TUNNEL_DMA,
 393				nhi_port, dst_port);
 394	else
 395		tb_tunnel_event(tb, TB_TUNNEL_DEACTIVATED, TB_TUNNEL_DMA,
 396				nhi_port, dst_port);
 397}
 398
 399static void icm_dp_event(struct tb *tb)
 400{
 401	tb_tunnel_event(tb, TB_TUNNEL_CHANGED, TB_TUNNEL_DP, NULL, NULL);
 402}
 403
 404static bool icm_fr_is_supported(struct tb *tb)
 405{
 406	return !x86_apple_machine;
 407}
 408
 409static inline int icm_fr_get_switch_index(u32 port)
 410{
 411	int index;
 412
 413	if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)
 414		return 0;
 415
 416	index = port >> ICM_PORT_INDEX_SHIFT;
 417	return index != 0xff ? index : 0;
 418}
 419
 420static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
 421{
 422	struct icm_fr_pkg_get_topology_response *switches, *sw;
 423	struct icm_fr_pkg_get_topology request = {
 424		.hdr = { .code = ICM_GET_TOPOLOGY },
 425	};
 426	size_t npackets = ICM_GET_TOPOLOGY_PACKETS;
 427	int ret, index;
 428	u8 i;
 429
 430	switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL);
 431	if (!switches)
 432		return -ENOMEM;
 433
 434	ret = icm_request(tb, &request, sizeof(request), switches,
 435			  sizeof(*switches), npackets, ICM_RETRIES, ICM_TIMEOUT);
 436	if (ret)
 437		goto err_free;
 438
 439	sw = &switches[0];
 440	index = icm_fr_get_switch_index(sw->ports[link]);
 441	if (!index) {
 442		ret = -ENODEV;
 443		goto err_free;
 444	}
 445
 446	sw = &switches[index];
 447	for (i = 1; i < depth; i++) {
 448		unsigned int j;
 449
 450		if (!(sw->first_data & ICM_SWITCH_USED)) {
 451			ret = -ENODEV;
 452			goto err_free;
 453		}
 454
 455		for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {
 456			index = icm_fr_get_switch_index(sw->ports[j]);
 457			if (index > sw->switch_index) {
 458				sw = &switches[index];
 459				break;
 460			}
 461		}
 462	}
 463
 464	*route = get_route(sw->route_hi, sw->route_lo);
 465
 466err_free:
 467	kfree(switches);
 468	return ret;
 469}
 470
 471static void icm_fr_save_devices(struct tb *tb)
 472{
 473	nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);
 474}
 475
 476static int
 477icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
 478		    u8 *proto_version, size_t *nboot_acl, bool *rpm)
 479{
 480	struct icm_fr_pkg_driver_ready_response reply;
 481	struct icm_pkg_driver_ready request = {
 482		.hdr.code = ICM_DRIVER_READY,
 483	};
 484	int ret;
 485
 486	memset(&reply, 0, sizeof(reply));
 487	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 488			  1, ICM_RETRIES, ICM_TIMEOUT);
 489	if (ret)
 490		return ret;
 491
 492	if (security_level)
 493		*security_level = reply.security_level & ICM_FR_SLEVEL_MASK;
 494
 495	return 0;
 496}
 497
 498static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)
 499{
 500	struct icm_fr_pkg_approve_device request;
 501	struct icm_fr_pkg_approve_device reply;
 502	int ret;
 503
 504	memset(&request, 0, sizeof(request));
 505	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 506	request.hdr.code = ICM_APPROVE_DEVICE;
 507	request.connection_id = sw->connection_id;
 508	request.connection_key = sw->connection_key;
 509
 510	memset(&reply, 0, sizeof(reply));
 511	/* Use larger timeout as establishing tunnels can take some time */
 512	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 513			  1, ICM_RETRIES, ICM_APPROVE_TIMEOUT);
 514	if (ret)
 515		return ret;
 516
 517	if (reply.hdr.flags & ICM_FLAGS_ERROR) {
 518		tb_warn(tb, "PCIe tunnel creation failed\n");
 519		return -EIO;
 520	}
 521
 522	return 0;
 523}
 524
 525static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)
 526{
 527	struct icm_fr_pkg_add_device_key request;
 528	struct icm_fr_pkg_add_device_key_response reply;
 529	int ret;
 530
 531	memset(&request, 0, sizeof(request));
 532	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 533	request.hdr.code = ICM_ADD_DEVICE_KEY;
 534	request.connection_id = sw->connection_id;
 535	request.connection_key = sw->connection_key;
 536	memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
 537
 538	memset(&reply, 0, sizeof(reply));
 539	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 540			  1, ICM_RETRIES, ICM_TIMEOUT);
 541	if (ret)
 542		return ret;
 543
 544	if (reply.hdr.flags & ICM_FLAGS_ERROR) {
 545		tb_warn(tb, "Adding key to switch failed\n");
 546		return -EIO;
 547	}
 548
 549	return 0;
 550}
 551
 552static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
 553				       const u8 *challenge, u8 *response)
 554{
 555	struct icm_fr_pkg_challenge_device request;
 556	struct icm_fr_pkg_challenge_device_response reply;
 557	int ret;
 558
 559	memset(&request, 0, sizeof(request));
 560	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 561	request.hdr.code = ICM_CHALLENGE_DEVICE;
 562	request.connection_id = sw->connection_id;
 563	request.connection_key = sw->connection_key;
 564	memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
 565
 566	memset(&reply, 0, sizeof(reply));
 567	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 568			  1, ICM_RETRIES, ICM_TIMEOUT);
 569	if (ret)
 570		return ret;
 571
 572	if (reply.hdr.flags & ICM_FLAGS_ERROR)
 573		return -EKEYREJECTED;
 574	if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
 575		return -ENOKEY;
 576
 577	memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
 578
 579	return 0;
 580}
 581
 582static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 583					int transmit_path, int transmit_ring,
 584					int receive_path, int receive_ring)
 585{
 586	struct icm_fr_pkg_approve_xdomain_response reply;
 587	struct icm_fr_pkg_approve_xdomain request;
 588	int ret;
 589
 590	memset(&request, 0, sizeof(request));
 591	request.hdr.code = ICM_APPROVE_XDOMAIN;
 592	request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
 593	memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
 594
 595	request.transmit_path = transmit_path;
 596	request.transmit_ring = transmit_ring;
 597	request.receive_path = receive_path;
 598	request.receive_ring = receive_ring;
 599
 600	memset(&reply, 0, sizeof(reply));
 601	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 602			  1, ICM_RETRIES, ICM_TIMEOUT);
 603	if (ret)
 604		return ret;
 605
 606	if (reply.hdr.flags & ICM_FLAGS_ERROR)
 607		return -EIO;
 608
 609	icm_xdomain_activated(xd, true);
 610	return 0;
 611}
 612
 613static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 614					   int transmit_path, int transmit_ring,
 615					   int receive_path, int receive_ring)
 616{
 617	u8 phy_port;
 618	u8 cmd;
 619
 620	phy_port = tb_phy_port_from_link(xd->link);
 621	if (phy_port == 0)
 622		cmd = NHI_MAILBOX_DISCONNECT_PA;
 623	else
 624		cmd = NHI_MAILBOX_DISCONNECT_PB;
 625
 626	nhi_mailbox_cmd(tb->nhi, cmd, 1);
 627	usleep_range(10, 50);
 628	nhi_mailbox_cmd(tb->nhi, cmd, 2);
 629
 630	icm_xdomain_activated(xd, false);
 631	return 0;
 632}
 633
 634static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,
 635				      const uuid_t *uuid)
 636{
 637	struct tb *tb = parent_sw->tb;
 638	struct tb_switch *sw;
 639
 640	sw = tb_switch_alloc(tb, &parent_sw->dev, route);
 641	if (IS_ERR(sw)) {
 642		tb_warn(tb, "failed to allocate switch at %llx\n", route);
 643		return sw;
 644	}
 645
 646	sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);
 647	if (!sw->uuid) {
 648		tb_switch_put(sw);
 649		return ERR_PTR(-ENOMEM);
 650	}
 651
 652	init_completion(&sw->rpm_complete);
 653	return sw;
 654}
 655
 656static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)
 657{
 658	u64 route = tb_route(sw);
 659	int ret;
 660
 661	/* Link the two switches now */
 662	tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
 663	tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);
 664
 665	ret = tb_switch_add(sw);
 666	if (ret)
 667		tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
 668
 669	return ret;
 670}
 671
 672static void update_switch(struct tb_switch *sw, u64 route, u8 connection_id,
 673			  u8 connection_key, u8 link, u8 depth, bool boot)
 674{
 675	struct tb_switch *parent_sw = tb_switch_parent(sw);
 676
 677	/* Disconnect from parent */
 678	tb_switch_downstream_port(sw)->remote = NULL;
 679	/* Re-connect via updated port */
 680	tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
 681
 682	/* Update with the new addressing information */
 683	sw->config.route_hi = upper_32_bits(route);
 684	sw->config.route_lo = lower_32_bits(route);
 685	sw->connection_id = connection_id;
 686	sw->connection_key = connection_key;
 687	sw->link = link;
 688	sw->depth = depth;
 689	sw->boot = boot;
 690
 691	/* This switch still exists */
 692	sw->is_unplugged = false;
 693
 694	/* Runtime resume is now complete */
 695	complete(&sw->rpm_complete);
 696}
 697
 698static void remove_switch(struct tb_switch *sw)
 699{
 700	tb_switch_downstream_port(sw)->remote = NULL;
 701	tb_switch_remove(sw);
 702}
 703
 704static void add_xdomain(struct tb_switch *sw, u64 route,
 705			const uuid_t *local_uuid, const uuid_t *remote_uuid,
 706			u8 link, u8 depth)
 707{
 708	struct tb_xdomain *xd;
 709
 710	pm_runtime_get_sync(&sw->dev);
 711
 712	xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);
 713	if (!xd)
 714		goto out;
 715
 716	xd->link = link;
 717	xd->depth = depth;
 718
 719	tb_port_at(route, sw)->xdomain = xd;
 720
 721	tb_xdomain_add(xd);
 722
 723out:
 724	pm_runtime_mark_last_busy(&sw->dev);
 725	pm_runtime_put_autosuspend(&sw->dev);
 726}
 727
 728static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)
 729{
 730	xd->link = link;
 731	xd->route = route;
 732	xd->is_unplugged = false;
 733}
 734
 735static void remove_xdomain(struct tb_xdomain *xd)
 736{
 737	struct tb_switch *sw;
 738
 739	sw = tb_to_switch(xd->dev.parent);
 740	tb_port_at(xd->route, sw)->xdomain = NULL;
 741	tb_xdomain_remove(xd);
 742}
 743
 744static void
 745icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
 746{
 747	const struct icm_fr_event_device_connected *pkg =
 748		(const struct icm_fr_event_device_connected *)hdr;
 749	enum tb_security_level security_level;
 750	struct tb_switch *sw, *parent_sw;
 751	bool boot, dual_lane, speed_gen3;
 752	struct icm *icm = tb_priv(tb);
 753	bool authorized = false;
 754	struct tb_xdomain *xd;
 755	u8 link, depth;
 756	u64 route;
 757	int ret;
 758
 759	icm_postpone_rescan(tb);
 760
 761	link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 762	depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 763		ICM_LINK_INFO_DEPTH_SHIFT;
 764	authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
 765	security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
 766			 ICM_FLAGS_SLEVEL_SHIFT;
 767	boot = pkg->link_info & ICM_LINK_INFO_BOOT;
 768	dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
 769	speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
 770
 771	if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
 772		tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",
 773			link, depth);
 774		return;
 775	}
 776
 777	sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
 778	if (sw) {
 779		u8 phy_port, sw_phy_port;
 780
 781		sw_phy_port = tb_phy_port_from_link(sw->link);
 782		phy_port = tb_phy_port_from_link(link);
 783
 784		/*
 785		 * On resume ICM will send us connected events for the
 786		 * devices that still are present. However, that
 787		 * information might have changed for example by the
 788		 * fact that a switch on a dual-link connection might
 789		 * have been enumerated using the other link now. Make
 790		 * sure our bookkeeping matches that.
 791		 */
 792		if (sw->depth == depth && sw_phy_port == phy_port &&
 793		    !!sw->authorized == authorized) {
 794			/*
 795			 * It was enumerated through another link so update
 796			 * route string accordingly.
 797			 */
 798			if (sw->link != link) {
 799				ret = icm->get_route(tb, link, depth, &route);
 800				if (ret) {
 801					tb_err(tb, "failed to update route string for switch at %u.%u\n",
 802					       link, depth);
 803					tb_switch_put(sw);
 804					return;
 805				}
 806			} else {
 807				route = tb_route(sw);
 808			}
 809
 810			update_switch(sw, route, pkg->connection_id,
 811				      pkg->connection_key, link, depth, boot);
 812			tb_switch_put(sw);
 813			return;
 814		}
 815
 816		/*
 817		 * User connected the same switch to another physical
 818		 * port or to another part of the topology. Remove the
 819		 * existing switch now before adding the new one.
 820		 */
 821		remove_switch(sw);
 822		tb_switch_put(sw);
 823	}
 824
 825	/*
 826	 * If the switch was not found by UUID, look for a switch on
 827	 * same physical port (taking possible link aggregation into
 828	 * account) and depth. If we found one it is definitely a stale
 829	 * one so remove it first.
 830	 */
 831	sw = tb_switch_find_by_link_depth(tb, link, depth);
 832	if (!sw) {
 833		u8 dual_link;
 834
 835		dual_link = dual_link_from_link(link);
 836		if (dual_link)
 837			sw = tb_switch_find_by_link_depth(tb, dual_link, depth);
 838	}
 839	if (sw) {
 840		remove_switch(sw);
 841		tb_switch_put(sw);
 842	}
 843
 844	/* Remove existing XDomain connection if found */
 845	xd = tb_xdomain_find_by_link_depth(tb, link, depth);
 846	if (xd) {
 847		remove_xdomain(xd);
 848		tb_xdomain_put(xd);
 849	}
 850
 851	parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);
 852	if (!parent_sw) {
 853		tb_err(tb, "failed to find parent switch for %u.%u\n",
 854		       link, depth);
 855		return;
 856	}
 857
 858	ret = icm->get_route(tb, link, depth, &route);
 859	if (ret) {
 860		tb_err(tb, "failed to find route string for switch at %u.%u\n",
 861		       link, depth);
 862		tb_switch_put(parent_sw);
 863		return;
 864	}
 865
 866	pm_runtime_get_sync(&parent_sw->dev);
 867
 868	sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
 869	if (!IS_ERR(sw)) {
 870		sw->connection_id = pkg->connection_id;
 871		sw->connection_key = pkg->connection_key;
 872		sw->link = link;
 873		sw->depth = depth;
 874		sw->authorized = authorized;
 875		sw->security_level = security_level;
 876		sw->boot = boot;
 877		sw->link_speed = speed_gen3 ? 20 : 10;
 878		sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :
 879					     TB_LINK_WIDTH_SINGLE;
 880		sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));
 881
 882		if (add_switch(parent_sw, sw))
 883			tb_switch_put(sw);
 884	}
 885
 886	pm_runtime_mark_last_busy(&parent_sw->dev);
 887	pm_runtime_put_autosuspend(&parent_sw->dev);
 888
 889	tb_switch_put(parent_sw);
 890}
 891
 892static void
 893icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
 894{
 895	const struct icm_fr_event_device_disconnected *pkg =
 896		(const struct icm_fr_event_device_disconnected *)hdr;
 897	struct tb_switch *sw;
 898	u8 link, depth;
 899
 900	link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 901	depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 902		ICM_LINK_INFO_DEPTH_SHIFT;
 903
 904	if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
 905		tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
 906		return;
 907	}
 908
 909	sw = tb_switch_find_by_link_depth(tb, link, depth);
 910	if (!sw) {
 911		tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
 912			depth);
 913		return;
 914	}
 915
 916	pm_runtime_get_sync(sw->dev.parent);
 917
 918	remove_switch(sw);
 919
 920	pm_runtime_mark_last_busy(sw->dev.parent);
 921	pm_runtime_put_autosuspend(sw->dev.parent);
 922
 923	tb_switch_put(sw);
 924}
 925
 926static void
 927icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
 928{
 929	const struct icm_fr_event_xdomain_connected *pkg =
 930		(const struct icm_fr_event_xdomain_connected *)hdr;
 931	struct tb_xdomain *xd;
 932	struct tb_switch *sw;
 933	u8 link, depth;
 934	u64 route;
 935
 936	link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 937	depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 938		ICM_LINK_INFO_DEPTH_SHIFT;
 939
 940	if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
 941		tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
 942		return;
 943	}
 944
 945	route = get_route(pkg->local_route_hi, pkg->local_route_lo);
 946
 947	xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
 948	if (xd) {
 949		u8 xd_phy_port, phy_port;
 950
 951		xd_phy_port = phy_port_from_route(xd->route, xd->depth);
 952		phy_port = phy_port_from_route(route, depth);
 953
 954		if (xd->depth == depth && xd_phy_port == phy_port) {
 955			update_xdomain(xd, route, link);
 956			tb_xdomain_put(xd);
 957			return;
 958		}
 959
 960		/*
 961		 * If we find an existing XDomain connection remove it
 962		 * now. We need to go through login handshake and
 963		 * everything anyway to be able to re-establish the
 964		 * connection.
 965		 */
 966		remove_xdomain(xd);
 967		tb_xdomain_put(xd);
 968	}
 969
 970	/*
 971	 * Look if there already exists an XDomain in the same place
 972	 * as the new one and in that case remove it because it is
 973	 * most likely another host that got disconnected.
 974	 */
 975	xd = tb_xdomain_find_by_link_depth(tb, link, depth);
 976	if (!xd) {
 977		u8 dual_link;
 978
 979		dual_link = dual_link_from_link(link);
 980		if (dual_link)
 981			xd = tb_xdomain_find_by_link_depth(tb, dual_link,
 982							   depth);
 983	}
 984	if (xd) {
 985		remove_xdomain(xd);
 986		tb_xdomain_put(xd);
 987	}
 988
 989	/*
 990	 * If the user disconnected a switch during suspend and
 991	 * connected another host to the same port, remove the switch
 992	 * first.
 993	 */
 994	sw = tb_switch_find_by_route(tb, route);
 995	if (sw) {
 996		remove_switch(sw);
 997		tb_switch_put(sw);
 998	}
 999
1000	sw = tb_switch_find_by_link_depth(tb, link, depth);
1001	if (!sw) {
1002		tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
1003			depth);
1004		return;
1005	}
1006
1007	add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,
1008		    depth);
1009	tb_switch_put(sw);
1010}
1011
1012static void
1013icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1014{
1015	const struct icm_fr_event_xdomain_disconnected *pkg =
1016		(const struct icm_fr_event_xdomain_disconnected *)hdr;
1017	struct tb_xdomain *xd;
1018
1019	/*
1020	 * If the connection is through one or multiple devices, the
1021	 * XDomain device is removed along with them so it is fine if we
1022	 * cannot find it here.
1023	 */
1024	xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1025	if (xd) {
1026		remove_xdomain(xd);
1027		tb_xdomain_put(xd);
1028	}
1029}
1030
1031static int icm_tr_cio_reset(struct tb *tb)
1032{
1033	return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));
1034}
1035
1036static int
1037icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1038		    u8 *proto_version, size_t *nboot_acl, bool *rpm)
1039{
1040	struct icm_tr_pkg_driver_ready_response reply;
1041	struct icm_pkg_driver_ready request = {
1042		.hdr.code = ICM_DRIVER_READY,
1043	};
1044	int ret;
1045
1046	memset(&reply, 0, sizeof(reply));
1047	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1048			  1, 10, 250);
1049	if (ret)
1050		return ret;
1051
1052	if (security_level)
1053		*security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;
1054	if (proto_version)
1055		*proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1056				ICM_TR_INFO_PROTO_VERSION_SHIFT;
1057	if (nboot_acl)
1058		*nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>
1059				ICM_TR_INFO_BOOT_ACL_SHIFT;
1060	if (rpm)
1061		*rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);
1062
1063	return 0;
1064}
1065
1066static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)
1067{
1068	struct icm_tr_pkg_approve_device request;
1069	struct icm_tr_pkg_approve_device reply;
1070	int ret;
1071
1072	memset(&request, 0, sizeof(request));
1073	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1074	request.hdr.code = ICM_APPROVE_DEVICE;
1075	request.route_lo = sw->config.route_lo;
1076	request.route_hi = sw->config.route_hi;
1077	request.connection_id = sw->connection_id;
1078
1079	memset(&reply, 0, sizeof(reply));
1080	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1081			  1, ICM_RETRIES, ICM_APPROVE_TIMEOUT);
1082	if (ret)
1083		return ret;
1084
1085	if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1086		tb_warn(tb, "PCIe tunnel creation failed\n");
1087		return -EIO;
1088	}
1089
1090	return 0;
1091}
1092
1093static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)
1094{
1095	struct icm_tr_pkg_add_device_key_response reply;
1096	struct icm_tr_pkg_add_device_key request;
1097	int ret;
1098
1099	memset(&request, 0, sizeof(request));
1100	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1101	request.hdr.code = ICM_ADD_DEVICE_KEY;
1102	request.route_lo = sw->config.route_lo;
1103	request.route_hi = sw->config.route_hi;
1104	request.connection_id = sw->connection_id;
1105	memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
1106
1107	memset(&reply, 0, sizeof(reply));
1108	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1109			  1, ICM_RETRIES, ICM_TIMEOUT);
1110	if (ret)
1111		return ret;
1112
1113	if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1114		tb_warn(tb, "Adding key to switch failed\n");
1115		return -EIO;
1116	}
1117
1118	return 0;
1119}
1120
1121static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
1122				       const u8 *challenge, u8 *response)
1123{
1124	struct icm_tr_pkg_challenge_device_response reply;
1125	struct icm_tr_pkg_challenge_device request;
1126	int ret;
1127
1128	memset(&request, 0, sizeof(request));
1129	memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1130	request.hdr.code = ICM_CHALLENGE_DEVICE;
1131	request.route_lo = sw->config.route_lo;
1132	request.route_hi = sw->config.route_hi;
1133	request.connection_id = sw->connection_id;
1134	memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
1135
1136	memset(&reply, 0, sizeof(reply));
1137	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1138			  1, ICM_RETRIES, ICM_TIMEOUT);
1139	if (ret)
1140		return ret;
1141
1142	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1143		return -EKEYREJECTED;
1144	if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
1145		return -ENOKEY;
1146
1147	memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
1148
1149	return 0;
1150}
1151
1152static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1153					int transmit_path, int transmit_ring,
1154					int receive_path, int receive_ring)
1155{
1156	struct icm_tr_pkg_approve_xdomain_response reply;
1157	struct icm_tr_pkg_approve_xdomain request;
1158	int ret;
1159
1160	memset(&request, 0, sizeof(request));
1161	request.hdr.code = ICM_APPROVE_XDOMAIN;
1162	request.route_hi = upper_32_bits(xd->route);
1163	request.route_lo = lower_32_bits(xd->route);
1164	request.transmit_path = transmit_path;
1165	request.transmit_ring = transmit_ring;
1166	request.receive_path = receive_path;
1167	request.receive_ring = receive_ring;
1168	memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1169
1170	memset(&reply, 0, sizeof(reply));
1171	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1172			  1, ICM_RETRIES, ICM_TIMEOUT);
1173	if (ret)
1174		return ret;
1175
1176	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1177		return -EIO;
1178
1179	icm_xdomain_activated(xd, true);
1180	return 0;
1181}
1182
1183static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,
1184				    int stage)
1185{
1186	struct icm_tr_pkg_disconnect_xdomain_response reply;
1187	struct icm_tr_pkg_disconnect_xdomain request;
1188	int ret;
1189
1190	memset(&request, 0, sizeof(request));
1191	request.hdr.code = ICM_DISCONNECT_XDOMAIN;
1192	request.stage = stage;
1193	request.route_hi = upper_32_bits(xd->route);
1194	request.route_lo = lower_32_bits(xd->route);
1195	memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1196
1197	memset(&reply, 0, sizeof(reply));
1198	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1199			  1, ICM_RETRIES, ICM_TIMEOUT);
1200	if (ret)
1201		return ret;
1202
1203	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1204		return -EIO;
1205
1206	return 0;
1207}
1208
1209static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1210					   int transmit_path, int transmit_ring,
1211					   int receive_path, int receive_ring)
1212{
1213	int ret;
1214
1215	ret = icm_tr_xdomain_tear_down(tb, xd, 1);
1216	if (ret)
1217		return ret;
1218
1219	usleep_range(10, 50);
1220	ret = icm_tr_xdomain_tear_down(tb, xd, 2);
1221	if (ret)
1222		return ret;
1223
1224	icm_xdomain_activated(xd, false);
1225	return 0;
1226}
1227
1228static void
1229__icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,
1230			  bool force_rtd3)
1231{
1232	const struct icm_tr_event_device_connected *pkg =
1233		(const struct icm_tr_event_device_connected *)hdr;
1234	bool authorized, boot, dual_lane, speed_gen3;
1235	enum tb_security_level security_level;
1236	struct tb_switch *sw, *parent_sw;
1237	struct tb_xdomain *xd;
1238	u64 route;
1239
1240	icm_postpone_rescan(tb);
1241
1242	/*
1243	 * Currently we don't use the QoS information coming with the
1244	 * device connected message so simply just ignore that extra
1245	 * packet for now.
1246	 */
1247	if (pkg->hdr.packet_id)
1248		return;
1249
1250	route = get_route(pkg->route_hi, pkg->route_lo);
1251	authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
1252	security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
1253			 ICM_FLAGS_SLEVEL_SHIFT;
1254	boot = pkg->link_info & ICM_LINK_INFO_BOOT;
1255	dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
1256	speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
1257
1258	if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
1259		tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",
1260			route);
1261		return;
1262	}
1263
1264	sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
1265	if (sw) {
1266		/* Update the switch if it is still in the same place */
1267		if (tb_route(sw) == route && !!sw->authorized == authorized) {
1268			update_switch(sw, route, pkg->connection_id, 0, 0, 0,
1269				      boot);
1270			tb_switch_put(sw);
1271			return;
1272		}
1273
1274		remove_switch(sw);
1275		tb_switch_put(sw);
1276	}
1277
1278	/* Another switch with the same address */
1279	sw = tb_switch_find_by_route(tb, route);
1280	if (sw) {
1281		remove_switch(sw);
1282		tb_switch_put(sw);
1283	}
1284
1285	/* XDomain connection with the same address */
1286	xd = tb_xdomain_find_by_route(tb, route);
1287	if (xd) {
1288		remove_xdomain(xd);
1289		tb_xdomain_put(xd);
1290	}
1291
1292	parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));
1293	if (!parent_sw) {
1294		tb_err(tb, "failed to find parent switch for %llx\n", route);
1295		return;
1296	}
1297
1298	pm_runtime_get_sync(&parent_sw->dev);
1299
1300	sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
1301	if (!IS_ERR(sw)) {
1302		sw->connection_id = pkg->connection_id;
1303		sw->authorized = authorized;
1304		sw->security_level = security_level;
1305		sw->boot = boot;
1306		sw->link_speed = speed_gen3 ? 20 : 10;
1307		sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :
1308					     TB_LINK_WIDTH_SINGLE;
1309		sw->rpm = force_rtd3;
1310		if (!sw->rpm)
1311			sw->rpm = intel_vss_is_rtd3(pkg->ep_name,
1312						    sizeof(pkg->ep_name));
1313
1314		if (add_switch(parent_sw, sw))
1315			tb_switch_put(sw);
1316	}
1317
1318	pm_runtime_mark_last_busy(&parent_sw->dev);
1319	pm_runtime_put_autosuspend(&parent_sw->dev);
1320
1321	tb_switch_put(parent_sw);
1322}
1323
1324static void
1325icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1326{
1327	__icm_tr_device_connected(tb, hdr, false);
1328}
1329
1330static void
1331icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1332{
1333	const struct icm_tr_event_device_disconnected *pkg =
1334		(const struct icm_tr_event_device_disconnected *)hdr;
1335	struct tb_switch *sw;
1336	u64 route;
1337
1338	route = get_route(pkg->route_hi, pkg->route_lo);
1339
1340	sw = tb_switch_find_by_route(tb, route);
1341	if (!sw) {
1342		tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1343		return;
1344	}
1345	pm_runtime_get_sync(sw->dev.parent);
1346
1347	remove_switch(sw);
1348
1349	pm_runtime_mark_last_busy(sw->dev.parent);
1350	pm_runtime_put_autosuspend(sw->dev.parent);
1351
1352	tb_switch_put(sw);
1353}
1354
1355static void
1356icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1357{
1358	const struct icm_tr_event_xdomain_connected *pkg =
1359		(const struct icm_tr_event_xdomain_connected *)hdr;
1360	struct tb_xdomain *xd;
1361	struct tb_switch *sw;
1362	u64 route;
1363
1364	if (!tb->root_switch)
1365		return;
1366
1367	route = get_route(pkg->local_route_hi, pkg->local_route_lo);
1368
1369	xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1370	if (xd) {
1371		if (xd->route == route) {
1372			update_xdomain(xd, route, 0);
1373			tb_xdomain_put(xd);
1374			return;
1375		}
1376
1377		remove_xdomain(xd);
1378		tb_xdomain_put(xd);
1379	}
1380
1381	/* An existing xdomain with the same address */
1382	xd = tb_xdomain_find_by_route(tb, route);
1383	if (xd) {
1384		remove_xdomain(xd);
1385		tb_xdomain_put(xd);
1386	}
1387
1388	/*
1389	 * If the user disconnected a switch during suspend and
1390	 * connected another host to the same port, remove the switch
1391	 * first.
1392	 */
1393	sw = tb_switch_find_by_route(tb, route);
1394	if (sw) {
1395		remove_switch(sw);
1396		tb_switch_put(sw);
1397	}
1398
1399	sw = tb_switch_find_by_route(tb, get_parent_route(route));
1400	if (!sw) {
1401		tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1402		return;
1403	}
1404
1405	add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);
1406	tb_switch_put(sw);
1407}
1408
1409static void
1410icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1411{
1412	const struct icm_tr_event_xdomain_disconnected *pkg =
1413		(const struct icm_tr_event_xdomain_disconnected *)hdr;
1414	struct tb_xdomain *xd;
1415	u64 route;
1416
1417	route = get_route(pkg->route_hi, pkg->route_lo);
1418
1419	xd = tb_xdomain_find_by_route(tb, route);
1420	if (xd) {
1421		remove_xdomain(xd);
1422		tb_xdomain_put(xd);
1423	}
1424}
1425
1426static struct pci_dev *get_upstream_port(struct pci_dev *pdev)
1427{
1428	struct pci_dev *parent;
1429
1430	parent = pci_upstream_bridge(pdev);
1431	while (parent) {
1432		if (!pci_is_pcie(parent))
1433			return NULL;
1434		if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)
1435			break;
1436		parent = pci_upstream_bridge(parent);
1437	}
1438
1439	if (!parent)
1440		return NULL;
1441
1442	switch (parent->device) {
1443	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:
1444	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:
1445	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:
1446	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:
1447	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:
1448	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:
1449	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:
1450		return parent;
1451	}
1452
1453	return NULL;
1454}
1455
1456static bool icm_ar_is_supported(struct tb *tb)
1457{
1458	struct pci_dev *upstream_port;
1459	struct icm *icm = tb_priv(tb);
1460
1461	/*
1462	 * Starting from Alpine Ridge we can use ICM on Apple machines
1463	 * as well. We just need to reset and re-enable it first.
1464	 * However, only start it if explicitly asked by the user.
1465	 */
1466	if (icm_firmware_running(tb->nhi))
1467		return true;
1468	if (!start_icm)
1469		return false;
1470
1471	/*
1472	 * Find the upstream PCIe port in case we need to do reset
1473	 * through its vendor specific registers.
1474	 */
1475	upstream_port = get_upstream_port(tb->nhi->pdev);
1476	if (upstream_port) {
1477		int cap;
1478
1479		cap = pci_find_ext_capability(upstream_port,
1480					      PCI_EXT_CAP_ID_VNDR);
1481		if (cap > 0) {
1482			icm->upstream_port = upstream_port;
1483			icm->vnd_cap = cap;
1484
1485			return true;
1486		}
1487	}
1488
1489	return false;
1490}
1491
1492static int icm_ar_cio_reset(struct tb *tb)
1493{
1494	return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));
1495}
1496
1497static int icm_ar_get_mode(struct tb *tb)
1498{
1499	struct tb_nhi *nhi = tb->nhi;
1500	int retries = 60;
1501	u32 val;
1502
1503	do {
1504		val = ioread32(nhi->iobase + REG_FW_STS);
1505		if (val & REG_FW_STS_NVM_AUTH_DONE)
1506			break;
1507		msleep(50);
1508	} while (--retries);
1509
1510	if (!retries) {
1511		dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n");
1512		return -ENODEV;
1513	}
1514
1515	return nhi_mailbox_mode(nhi);
1516}
1517
1518static int
1519icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1520		    u8 *proto_version, size_t *nboot_acl, bool *rpm)
1521{
1522	struct icm_ar_pkg_driver_ready_response reply;
1523	struct icm_pkg_driver_ready request = {
1524		.hdr.code = ICM_DRIVER_READY,
1525	};
1526	int ret;
1527
1528	memset(&reply, 0, sizeof(reply));
1529	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1530			  1, ICM_RETRIES, ICM_TIMEOUT);
1531	if (ret)
1532		return ret;
1533
1534	if (security_level)
1535		*security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;
1536	if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))
1537		*nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>
1538				ICM_AR_INFO_BOOT_ACL_SHIFT;
1539	if (rpm)
1540		*rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);
1541
1542	return 0;
1543}
1544
1545static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
1546{
1547	struct icm_ar_pkg_get_route_response reply;
1548	struct icm_ar_pkg_get_route request = {
1549		.hdr = { .code = ICM_GET_ROUTE },
1550		.link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,
1551	};
1552	int ret;
1553
1554	memset(&reply, 0, sizeof(reply));
1555	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1556			  1, ICM_RETRIES, ICM_TIMEOUT);
1557	if (ret)
1558		return ret;
1559
1560	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1561		return -EIO;
1562
1563	*route = get_route(reply.route_hi, reply.route_lo);
1564	return 0;
1565}
1566
1567static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)
1568{
1569	struct icm_ar_pkg_preboot_acl_response reply;
1570	struct icm_ar_pkg_preboot_acl request = {
1571		.hdr = { .code = ICM_PREBOOT_ACL },
1572	};
1573	int ret, i;
1574
1575	memset(&reply, 0, sizeof(reply));
1576	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1577			  1, ICM_RETRIES, ICM_TIMEOUT);
1578	if (ret)
1579		return ret;
1580
1581	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1582		return -EIO;
1583
1584	for (i = 0; i < nuuids; i++) {
1585		u32 *uuid = (u32 *)&uuids[i];
1586
1587		uuid[0] = reply.acl[i].uuid_lo;
1588		uuid[1] = reply.acl[i].uuid_hi;
1589
1590		if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {
1591			/* Map empty entries to null UUID */
1592			uuid[0] = 0;
1593			uuid[1] = 0;
1594		} else if (uuid[0] != 0 || uuid[1] != 0) {
1595			/* Upper two DWs are always one's */
1596			uuid[2] = 0xffffffff;
1597			uuid[3] = 0xffffffff;
1598		}
1599	}
1600
1601	return ret;
1602}
1603
1604static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,
1605			       size_t nuuids)
1606{
1607	struct icm_ar_pkg_preboot_acl_response reply;
1608	struct icm_ar_pkg_preboot_acl request = {
1609		.hdr = {
1610			.code = ICM_PREBOOT_ACL,
1611			.flags = ICM_FLAGS_WRITE,
1612		},
1613	};
1614	int ret, i;
1615
1616	for (i = 0; i < nuuids; i++) {
1617		const u32 *uuid = (const u32 *)&uuids[i];
1618
1619		if (uuid_is_null(&uuids[i])) {
1620			/*
1621			 * Map null UUID to the empty (all one) entries
1622			 * for ICM.
1623			 */
1624			request.acl[i].uuid_lo = 0xffffffff;
1625			request.acl[i].uuid_hi = 0xffffffff;
1626		} else {
1627			/* Two high DWs need to be set to all one */
1628			if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)
1629				return -EINVAL;
1630
1631			request.acl[i].uuid_lo = uuid[0];
1632			request.acl[i].uuid_hi = uuid[1];
1633		}
1634	}
1635
1636	memset(&reply, 0, sizeof(reply));
1637	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1638			  1, ICM_RETRIES, ICM_TIMEOUT);
1639	if (ret)
1640		return ret;
1641
1642	if (reply.hdr.flags & ICM_FLAGS_ERROR)
1643		return -EIO;
1644
1645	return 0;
1646}
1647
1648static int
1649icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1650		     u8 *proto_version, size_t *nboot_acl, bool *rpm)
1651{
1652	struct icm_tr_pkg_driver_ready_response reply;
1653	struct icm_pkg_driver_ready request = {
1654		.hdr.code = ICM_DRIVER_READY,
1655	};
1656	int ret;
1657
1658	memset(&reply, 0, sizeof(reply));
1659	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1660			  1, ICM_RETRIES, 20000);
1661	if (ret)
1662		return ret;
1663
1664	if (proto_version)
1665		*proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1666				ICM_TR_INFO_PROTO_VERSION_SHIFT;
1667
1668	/* Ice Lake always supports RTD3 */
1669	if (rpm)
1670		*rpm = true;
1671
1672	return 0;
1673}
1674
1675static void icm_icl_set_uuid(struct tb *tb)
1676{
1677	struct tb_nhi *nhi = tb->nhi;
1678	u32 uuid[4];
1679
1680	pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]);
1681	pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]);
1682	uuid[2] = 0xffffffff;
1683	uuid[3] = 0xffffffff;
1684
1685	tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
1686}
1687
1688static void
1689icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1690{
1691	__icm_tr_device_connected(tb, hdr, true);
1692}
1693
1694static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
1695{
1696	const struct icm_icl_event_rtd3_veto *pkg =
1697		(const struct icm_icl_event_rtd3_veto *)hdr;
1698
1699	tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);
1700
1701	if (pkg->veto_reason)
1702		icm_veto_begin(tb);
1703	else
1704		icm_veto_end(tb);
1705}
1706
1707static bool icm_tgl_is_supported(struct tb *tb)
1708{
1709	unsigned long end = jiffies + msecs_to_jiffies(10);
1710
1711	do {
1712		u32 val;
1713
1714		val = ioread32(tb->nhi->iobase + REG_FW_STS);
1715		if (val & REG_FW_STS_NVM_AUTH_DONE)
1716			return true;
1717		usleep_range(100, 500);
1718	} while (time_before(jiffies, end));
1719
1720	return false;
1721}
1722
1723static void icm_handle_notification(struct work_struct *work)
1724{
1725	struct icm_notification *n = container_of(work, typeof(*n), work);
1726	struct tb *tb = n->tb;
1727	struct icm *icm = tb_priv(tb);
1728
1729	mutex_lock(&tb->lock);
1730
1731	/*
1732	 * When the domain is stopped we flush its workqueue but before
1733	 * that the root switch is removed. In that case we should treat
1734	 * the queued events as being canceled.
1735	 */
1736	if (tb->root_switch) {
1737		switch (n->pkg->code) {
1738		case ICM_EVENT_DEVICE_CONNECTED:
1739			icm->device_connected(tb, n->pkg);
1740			break;
1741		case ICM_EVENT_DEVICE_DISCONNECTED:
1742			icm->device_disconnected(tb, n->pkg);
1743			break;
1744		case ICM_EVENT_XDOMAIN_CONNECTED:
1745			if (tb_is_xdomain_enabled())
1746				icm->xdomain_connected(tb, n->pkg);
1747			break;
1748		case ICM_EVENT_XDOMAIN_DISCONNECTED:
1749			if (tb_is_xdomain_enabled())
1750				icm->xdomain_disconnected(tb, n->pkg);
1751			break;
1752		case ICM_EVENT_DP_CONFIG_CHANGED:
1753			icm_dp_event(tb);
1754			break;
1755		case ICM_EVENT_RTD3_VETO:
1756			icm->rtd3_veto(tb, n->pkg);
1757			break;
1758		}
1759	}
1760
1761	mutex_unlock(&tb->lock);
1762
1763	kfree(n->pkg);
1764	kfree(n);
1765}
1766
1767static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1768			     const void *buf, size_t size)
1769{
1770	struct icm_notification *n;
1771
1772	n = kmalloc(sizeof(*n), GFP_KERNEL);
1773	if (!n)
1774		return;
1775
1776	n->pkg = kmemdup(buf, size, GFP_KERNEL);
1777	if (!n->pkg) {
1778		kfree(n);
1779		return;
1780	}
1781
1782	INIT_WORK(&n->work, icm_handle_notification);
1783	n->tb = tb;
1784
1785	queue_work(tb->wq, &n->work);
1786}
1787
1788static int
1789__icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1790		   u8 *proto_version, size_t *nboot_acl, bool *rpm)
1791{
1792	struct icm *icm = tb_priv(tb);
1793	unsigned int retries = 50;
1794	int ret;
1795
1796	ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,
1797				rpm);
1798	if (ret) {
1799		tb_err(tb, "failed to send driver ready to ICM\n");
1800		return ret;
1801	}
1802
1803	/*
1804	 * Hold on here until the switch config space is accessible so
1805	 * that we can read root switch config successfully.
1806	 */
1807	do {
1808		struct tb_cfg_result res;
1809		u32 tmp;
1810
1811		res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,
1812				      0, 1, 100);
1813		if (!res.err)
1814			return 0;
1815
1816		msleep(50);
1817	} while (--retries);
1818
1819	tb_err(tb, "failed to read root switch config space, giving up\n");
1820	return -ETIMEDOUT;
1821}
1822
1823static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)
1824{
1825	struct icm *icm = tb_priv(tb);
1826	u32 val;
1827
1828	if (!icm->upstream_port)
1829		return -ENODEV;
1830
1831	/* Put ARC to wait for CIO reset event to happen */
1832	val = ioread32(nhi->iobase + REG_FW_STS);
1833	val |= REG_FW_STS_CIO_RESET_REQ;
1834	iowrite32(val, nhi->iobase + REG_FW_STS);
1835
1836	/* Re-start ARC */
1837	val = ioread32(nhi->iobase + REG_FW_STS);
1838	val |= REG_FW_STS_ICM_EN_INVERT;
1839	val |= REG_FW_STS_ICM_EN_CPU;
1840	iowrite32(val, nhi->iobase + REG_FW_STS);
1841
1842	/* Trigger CIO reset now */
1843	return icm->cio_reset(tb);
1844}
1845
1846static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)
1847{
1848	unsigned int retries = 10;
1849	int ret;
1850	u32 val;
1851
1852	/* Check if the ICM firmware is already running */
1853	if (icm_firmware_running(nhi))
1854		return 0;
1855
1856	dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n");
1857
1858	ret = icm_firmware_reset(tb, nhi);
1859	if (ret)
1860		return ret;
1861
1862	/* Wait until the ICM firmware tells us it is up and running */
1863	do {
1864		/* Check that the ICM firmware is running */
1865		val = ioread32(nhi->iobase + REG_FW_STS);
1866		if (val & REG_FW_STS_NVM_AUTH_DONE)
1867			return 0;
1868
1869		msleep(300);
1870	} while (--retries);
1871
1872	return -ETIMEDOUT;
1873}
1874
1875static int icm_reset_phy_port(struct tb *tb, int phy_port)
1876{
1877	struct icm *icm = tb_priv(tb);
1878	u32 state0, state1;
1879	int port0, port1;
1880	u32 val0, val1;
1881	int ret;
1882
1883	if (!icm->upstream_port)
1884		return 0;
1885
1886	if (phy_port) {
1887		port0 = 3;
1888		port1 = 4;
1889	} else {
1890		port0 = 1;
1891		port1 = 2;
1892	}
1893
1894	/*
1895	 * Read link status of both null ports belonging to a single
1896	 * physical port.
1897	 */
1898	ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1899	if (ret)
1900		return ret;
1901	ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1902	if (ret)
1903		return ret;
1904
1905	state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;
1906	state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1907	state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;
1908	state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1909
1910	/* If they are both up we need to reset them now */
1911	if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)
1912		return 0;
1913
1914	val0 |= PHY_PORT_CS1_LINK_DISABLE;
1915	ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1916	if (ret)
1917		return ret;
1918
1919	val1 |= PHY_PORT_CS1_LINK_DISABLE;
1920	ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1921	if (ret)
1922		return ret;
1923
1924	/* Wait a bit and then re-enable both ports */
1925	usleep_range(10, 100);
1926
1927	ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1928	if (ret)
1929		return ret;
1930	ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1931	if (ret)
1932		return ret;
1933
1934	val0 &= ~PHY_PORT_CS1_LINK_DISABLE;
1935	ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1936	if (ret)
1937		return ret;
1938
1939	val1 &= ~PHY_PORT_CS1_LINK_DISABLE;
1940	return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1941}
1942
1943static int icm_firmware_init(struct tb *tb)
1944{
1945	struct icm *icm = tb_priv(tb);
1946	struct tb_nhi *nhi = tb->nhi;
1947	int ret;
1948
1949	ret = icm_firmware_start(tb, nhi);
1950	if (ret) {
1951		dev_err(&nhi->pdev->dev, "could not start ICM firmware\n");
1952		return ret;
1953	}
1954
1955	if (icm->get_mode) {
1956		ret = icm->get_mode(tb);
1957
1958		switch (ret) {
1959		case NHI_FW_SAFE_MODE:
1960			icm->safe_mode = true;
1961			break;
1962
1963		case NHI_FW_CM_MODE:
1964			/* Ask ICM to accept all Thunderbolt devices */
1965			nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);
1966			break;
1967
1968		default:
1969			if (ret < 0)
1970				return ret;
1971
1972			tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);
1973			return -ENODEV;
1974		}
1975	}
1976
1977	/*
1978	 * Reset both physical ports if there is anything connected to
1979	 * them already.
1980	 */
1981	ret = icm_reset_phy_port(tb, 0);
1982	if (ret)
1983		dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n");
1984	ret = icm_reset_phy_port(tb, 1);
1985	if (ret)
1986		dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n");
1987
1988	return 0;
1989}
1990
1991static int icm_driver_ready(struct tb *tb)
1992{
1993	struct icm *icm = tb_priv(tb);
1994	int ret;
1995
1996	ret = icm_firmware_init(tb);
1997	if (ret)
1998		return ret;
1999
2000	if (icm->safe_mode) {
2001		tb_info(tb, "Thunderbolt host controller is in safe mode.\n");
2002		tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");
2003		tb_info(tb, "Use fwupd tool to apply update. Check Documentation/admin-guide/thunderbolt.rst for details.\n");
2004		return 0;
2005	}
2006
2007	ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,
2008				 &tb->nboot_acl, &icm->rpm);
2009	if (ret)
2010		return ret;
2011
2012	/*
2013	 * Make sure the number of supported preboot ACL matches what we
2014	 * expect or disable the whole feature.
2015	 */
2016	if (tb->nboot_acl > icm->max_boot_acl)
2017		tb->nboot_acl = 0;
2018
2019	if (icm->proto_version >= 3)
2020		tb_dbg(tb, "USB4 proxy operations supported\n");
2021
2022	return 0;
2023}
2024
2025static int icm_suspend(struct tb *tb)
2026{
2027	struct icm *icm = tb_priv(tb);
2028
2029	if (icm->save_devices)
2030		icm->save_devices(tb);
2031
2032	nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2033	return 0;
2034}
2035
2036/*
2037 * Mark all switches (except root switch) below this one unplugged. ICM
2038 * firmware will send us an updated list of switches after we have send
2039 * it driver ready command. If a switch is not in that list it will be
2040 * removed when we perform rescan.
2041 */
2042static void icm_unplug_children(struct tb_switch *sw)
2043{
2044	struct tb_port *port;
2045
2046	if (tb_route(sw))
2047		sw->is_unplugged = true;
2048
2049	tb_switch_for_each_port(sw, port) {
2050		if (port->xdomain)
2051			port->xdomain->is_unplugged = true;
2052		else if (tb_port_has_remote(port))
2053			icm_unplug_children(port->remote->sw);
2054	}
2055}
2056
2057static int complete_rpm(struct device *dev, void *data)
2058{
2059	struct tb_switch *sw = tb_to_switch(dev);
2060
2061	if (sw)
2062		complete(&sw->rpm_complete);
2063	return 0;
2064}
2065
2066static void remove_unplugged_switch(struct tb_switch *sw)
2067{
2068	struct device *parent = get_device(sw->dev.parent);
2069
2070	pm_runtime_get_sync(parent);
2071
2072	/*
2073	 * Signal this and switches below for rpm_complete because
2074	 * tb_switch_remove() calls pm_runtime_get_sync() that then waits
2075	 * for it.
2076	 */
2077	complete_rpm(&sw->dev, NULL);
2078	bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);
2079	tb_switch_remove(sw);
2080
2081	pm_runtime_mark_last_busy(parent);
2082	pm_runtime_put_autosuspend(parent);
2083
2084	put_device(parent);
2085}
2086
2087static void icm_free_unplugged_children(struct tb_switch *sw)
2088{
2089	struct tb_port *port;
2090
2091	tb_switch_for_each_port(sw, port) {
2092		if (port->xdomain && port->xdomain->is_unplugged) {
2093			tb_xdomain_remove(port->xdomain);
2094			port->xdomain = NULL;
2095		} else if (tb_port_has_remote(port)) {
2096			if (port->remote->sw->is_unplugged) {
2097				remove_unplugged_switch(port->remote->sw);
2098				port->remote = NULL;
2099			} else {
2100				icm_free_unplugged_children(port->remote->sw);
2101			}
2102		}
2103	}
2104}
2105
2106static void icm_rescan_work(struct work_struct *work)
2107{
2108	struct icm *icm = container_of(work, struct icm, rescan_work.work);
2109	struct tb *tb = icm_to_tb(icm);
2110
2111	mutex_lock(&tb->lock);
2112	if (tb->root_switch)
2113		icm_free_unplugged_children(tb->root_switch);
2114	mutex_unlock(&tb->lock);
2115}
2116
2117static void icm_complete(struct tb *tb)
2118{
2119	struct icm *icm = tb_priv(tb);
2120
2121	if (tb->nhi->going_away)
2122		return;
2123
2124	/*
2125	 * If RTD3 was vetoed before we entered system suspend allow it
2126	 * again now before driver ready is sent. Firmware sends a new RTD3
2127	 * veto if it is still the case after we have sent it driver ready
2128	 * command.
2129	 */
2130	icm_veto_end(tb);
2131	icm_unplug_children(tb->root_switch);
2132
2133	/*
2134	 * Now all existing children should be resumed, start events
2135	 * from ICM to get updated status.
2136	 */
2137	__icm_driver_ready(tb, NULL, NULL, NULL, NULL);
2138
2139	/*
2140	 * We do not get notifications of devices that have been
2141	 * unplugged during suspend so schedule rescan to clean them up
2142	 * if any.
2143	 */
2144	queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));
2145}
2146
2147static int icm_runtime_suspend(struct tb *tb)
2148{
2149	nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2150	return 0;
2151}
2152
2153static int icm_runtime_suspend_switch(struct tb_switch *sw)
2154{
2155	if (tb_route(sw))
2156		reinit_completion(&sw->rpm_complete);
2157	return 0;
2158}
2159
2160static int icm_runtime_resume_switch(struct tb_switch *sw)
2161{
2162	if (tb_route(sw)) {
2163		if (!wait_for_completion_timeout(&sw->rpm_complete,
2164						 msecs_to_jiffies(500))) {
2165			dev_dbg(&sw->dev, "runtime resuming timed out\n");
2166		}
2167	}
2168	return 0;
2169}
2170
2171static int icm_runtime_resume(struct tb *tb)
2172{
2173	/*
2174	 * We can reuse the same resume functionality as with system
2175	 * suspend.
2176	 */
2177	icm_complete(tb);
2178	return 0;
2179}
2180
2181static int icm_start(struct tb *tb, bool not_used)
2182{
2183	struct icm *icm = tb_priv(tb);
2184	int ret;
2185
2186	if (icm->safe_mode)
2187		tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);
2188	else
2189		tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
2190	if (IS_ERR(tb->root_switch))
2191		return PTR_ERR(tb->root_switch);
2192
2193	tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;
2194	tb->root_switch->rpm = icm->rpm;
2195
2196	if (icm->set_uuid)
2197		icm->set_uuid(tb);
2198
2199	ret = tb_switch_add(tb->root_switch);
2200	if (ret) {
2201		tb_switch_put(tb->root_switch);
2202		tb->root_switch = NULL;
2203	}
2204
2205	return ret;
2206}
2207
2208static void icm_stop(struct tb *tb)
2209{
2210	struct icm *icm = tb_priv(tb);
2211
2212	cancel_delayed_work(&icm->rescan_work);
2213	tb_switch_remove(tb->root_switch);
2214	tb->root_switch = NULL;
2215	nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2216	kfree(icm->last_nvm_auth);
2217	icm->last_nvm_auth = NULL;
2218}
2219
2220static int icm_disconnect_pcie_paths(struct tb *tb)
2221{
2222	return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);
2223}
2224
2225static void icm_usb4_switch_nvm_auth_complete(void *data)
2226{
2227	struct usb4_switch_nvm_auth *auth = data;
2228	struct icm *icm = auth->icm;
2229	struct tb *tb = icm_to_tb(icm);
2230
2231	tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",
2232	       get_route(auth->reply.route_hi, auth->reply.route_lo),
2233	       auth->reply.hdr.flags, auth->reply.status);
2234
2235	mutex_lock(&tb->lock);
2236	if (WARN_ON(icm->last_nvm_auth))
2237		kfree(icm->last_nvm_auth);
2238	icm->last_nvm_auth = auth;
2239	mutex_unlock(&tb->lock);
2240}
2241
2242static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)
2243{
2244	struct usb4_switch_nvm_auth *auth;
2245	struct icm *icm = tb_priv(tb);
2246	struct tb_cfg_request *req;
2247	int ret;
2248
2249	auth = kzalloc(sizeof(*auth), GFP_KERNEL);
2250	if (!auth)
2251		return -ENOMEM;
2252
2253	auth->icm = icm;
2254	auth->request.hdr.code = ICM_USB4_SWITCH_OP;
2255	auth->request.route_hi = upper_32_bits(route);
2256	auth->request.route_lo = lower_32_bits(route);
2257	auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;
2258
2259	req = tb_cfg_request_alloc();
2260	if (!req) {
2261		ret = -ENOMEM;
2262		goto err_free_auth;
2263	}
2264
2265	req->match = icm_match;
2266	req->copy = icm_copy;
2267	req->request = &auth->request;
2268	req->request_size = sizeof(auth->request);
2269	req->request_type = TB_CFG_PKG_ICM_CMD;
2270	req->response = &auth->reply;
2271	req->npackets = 1;
2272	req->response_size = sizeof(auth->reply);
2273	req->response_type = TB_CFG_PKG_ICM_RESP;
2274
2275	tb_dbg(tb, "NVM_AUTH request for %llx\n", route);
2276
2277	mutex_lock(&icm->request_lock);
2278	ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,
2279			     auth);
2280	mutex_unlock(&icm->request_lock);
2281
2282	tb_cfg_request_put(req);
2283	if (ret)
2284		goto err_free_auth;
2285	return 0;
2286
2287err_free_auth:
2288	kfree(auth);
2289	return ret;
2290}
2291
2292static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
2293			      u8 *status, const void *tx_data, size_t tx_data_len,
2294			      void *rx_data, size_t rx_data_len)
2295{
2296	struct icm_usb4_switch_op_response reply;
2297	struct icm_usb4_switch_op request;
2298	struct tb *tb = sw->tb;
2299	struct icm *icm = tb_priv(tb);
2300	u64 route = tb_route(sw);
2301	int ret;
2302
2303	/*
2304	 * USB4 router operation proxy is supported in firmware if the
2305	 * protocol version is 3 or higher.
2306	 */
2307	if (icm->proto_version < 3)
2308		return -EOPNOTSUPP;
2309
2310	/*
2311	 * NVM_AUTH is a special USB4 proxy operation that does not
2312	 * return immediately so handle it separately.
2313	 */
2314	if (opcode == USB4_SWITCH_OP_NVM_AUTH)
2315		return icm_usb4_switch_nvm_authenticate(tb, route);
2316
2317	memset(&request, 0, sizeof(request));
2318	request.hdr.code = ICM_USB4_SWITCH_OP;
2319	request.route_hi = upper_32_bits(route);
2320	request.route_lo = lower_32_bits(route);
2321	request.opcode = opcode;
2322	if (metadata)
2323		request.metadata = *metadata;
2324
2325	if (tx_data_len) {
2326		request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;
2327		if (tx_data_len < ARRAY_SIZE(request.data))
2328			request.data_len_valid =
2329				tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;
2330		memcpy(request.data, tx_data, tx_data_len * sizeof(u32));
2331	}
2332
2333	memset(&reply, 0, sizeof(reply));
2334	ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
2335			  1, ICM_RETRIES, ICM_TIMEOUT);
2336	if (ret)
2337		return ret;
2338
2339	if (reply.hdr.flags & ICM_FLAGS_ERROR)
2340		return -EIO;
2341
2342	if (status)
2343		*status = reply.status;
2344
2345	if (metadata)
2346		*metadata = reply.metadata;
2347
2348	if (rx_data_len)
2349		memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));
2350
2351	return 0;
2352}
2353
2354static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,
2355						   u32 *status)
2356{
2357	struct usb4_switch_nvm_auth *auth;
2358	struct tb *tb = sw->tb;
2359	struct icm *icm = tb_priv(tb);
2360	int ret = 0;
2361
2362	if (icm->proto_version < 3)
2363		return -EOPNOTSUPP;
2364
2365	auth = icm->last_nvm_auth;
2366	icm->last_nvm_auth = NULL;
2367
2368	if (auth && auth->reply.route_hi == sw->config.route_hi &&
2369	    auth->reply.route_lo == sw->config.route_lo) {
2370		tb_dbg(tb, "NVM_AUTH found for %llx flags %#x status %#x\n",
2371		       tb_route(sw), auth->reply.hdr.flags, auth->reply.status);
2372		if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)
2373			ret = -EIO;
2374		else
2375			*status = auth->reply.status;
2376	} else {
2377		*status = 0;
2378	}
2379
2380	kfree(auth);
2381	return ret;
2382}
2383
2384/* Falcon Ridge */
2385static const struct tb_cm_ops icm_fr_ops = {
2386	.driver_ready = icm_driver_ready,
2387	.start = icm_start,
2388	.stop = icm_stop,
2389	.suspend = icm_suspend,
2390	.complete = icm_complete,
2391	.handle_event = icm_handle_event,
2392	.approve_switch = icm_fr_approve_switch,
2393	.add_switch_key = icm_fr_add_switch_key,
2394	.challenge_switch_key = icm_fr_challenge_switch_key,
2395	.disconnect_pcie_paths = icm_disconnect_pcie_paths,
2396	.approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2397	.disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2398};
2399
2400/* Alpine Ridge */
2401static const struct tb_cm_ops icm_ar_ops = {
2402	.driver_ready = icm_driver_ready,
2403	.start = icm_start,
2404	.stop = icm_stop,
2405	.suspend = icm_suspend,
2406	.complete = icm_complete,
2407	.runtime_suspend = icm_runtime_suspend,
2408	.runtime_resume = icm_runtime_resume,
2409	.runtime_suspend_switch = icm_runtime_suspend_switch,
2410	.runtime_resume_switch = icm_runtime_resume_switch,
2411	.handle_event = icm_handle_event,
2412	.get_boot_acl = icm_ar_get_boot_acl,
2413	.set_boot_acl = icm_ar_set_boot_acl,
2414	.approve_switch = icm_fr_approve_switch,
2415	.add_switch_key = icm_fr_add_switch_key,
2416	.challenge_switch_key = icm_fr_challenge_switch_key,
2417	.disconnect_pcie_paths = icm_disconnect_pcie_paths,
2418	.approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2419	.disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2420};
2421
2422/* Titan Ridge */
2423static const struct tb_cm_ops icm_tr_ops = {
2424	.driver_ready = icm_driver_ready,
2425	.start = icm_start,
2426	.stop = icm_stop,
2427	.suspend = icm_suspend,
2428	.complete = icm_complete,
2429	.runtime_suspend = icm_runtime_suspend,
2430	.runtime_resume = icm_runtime_resume,
2431	.runtime_suspend_switch = icm_runtime_suspend_switch,
2432	.runtime_resume_switch = icm_runtime_resume_switch,
2433	.handle_event = icm_handle_event,
2434	.get_boot_acl = icm_ar_get_boot_acl,
2435	.set_boot_acl = icm_ar_set_boot_acl,
2436	.approve_switch = icm_tr_approve_switch,
2437	.add_switch_key = icm_tr_add_switch_key,
2438	.challenge_switch_key = icm_tr_challenge_switch_key,
2439	.disconnect_pcie_paths = icm_disconnect_pcie_paths,
2440	.approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2441	.disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2442	.usb4_switch_op = icm_usb4_switch_op,
2443	.usb4_switch_nvm_authenticate_status =
2444		icm_usb4_switch_nvm_authenticate_status,
2445};
2446
2447/* Ice Lake */
2448static const struct tb_cm_ops icm_icl_ops = {
2449	.driver_ready = icm_driver_ready,
2450	.start = icm_start,
2451	.stop = icm_stop,
2452	.complete = icm_complete,
2453	.runtime_suspend = icm_runtime_suspend,
2454	.runtime_resume = icm_runtime_resume,
2455	.handle_event = icm_handle_event,
2456	.approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2457	.disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2458	.usb4_switch_op = icm_usb4_switch_op,
2459	.usb4_switch_nvm_authenticate_status =
2460		icm_usb4_switch_nvm_authenticate_status,
2461};
2462
2463struct tb *icm_probe(struct tb_nhi *nhi)
2464{
2465	struct icm *icm;
2466	struct tb *tb;
2467
2468	tb = tb_domain_alloc(nhi, ICM_TIMEOUT, sizeof(struct icm));
2469	if (!tb)
2470		return NULL;
2471
2472	icm = tb_priv(tb);
2473	INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);
2474	mutex_init(&icm->request_lock);
2475
2476	switch (nhi->pdev->device) {
2477	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
2478	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
2479		icm->can_upgrade_nvm = true;
2480		icm->is_supported = icm_fr_is_supported;
2481		icm->get_route = icm_fr_get_route;
2482		icm->save_devices = icm_fr_save_devices;
2483		icm->driver_ready = icm_fr_driver_ready;
2484		icm->device_connected = icm_fr_device_connected;
2485		icm->device_disconnected = icm_fr_device_disconnected;
2486		icm->xdomain_connected = icm_fr_xdomain_connected;
2487		icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2488		tb->cm_ops = &icm_fr_ops;
2489		break;
2490
2491	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:
2492	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:
2493	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:
2494	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:
2495	case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:
2496		icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2497		/*
2498		 * NVM upgrade has not been tested on Apple systems and
2499		 * they don't provide images publicly either. To be on
2500		 * the safe side prevent root switch NVM upgrade on Macs
2501		 * for now.
2502		 */
2503		icm->can_upgrade_nvm = !x86_apple_machine;
2504		icm->is_supported = icm_ar_is_supported;
2505		icm->cio_reset = icm_ar_cio_reset;
2506		icm->get_mode = icm_ar_get_mode;
2507		icm->get_route = icm_ar_get_route;
2508		icm->save_devices = icm_fr_save_devices;
2509		icm->driver_ready = icm_ar_driver_ready;
2510		icm->device_connected = icm_fr_device_connected;
2511		icm->device_disconnected = icm_fr_device_disconnected;
2512		icm->xdomain_connected = icm_fr_xdomain_connected;
2513		icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2514		tb->cm_ops = &icm_ar_ops;
2515		break;
2516
2517	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
2518	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
2519		icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2520		icm->can_upgrade_nvm = !x86_apple_machine;
2521		icm->is_supported = icm_ar_is_supported;
2522		icm->cio_reset = icm_tr_cio_reset;
2523		icm->get_mode = icm_ar_get_mode;
2524		icm->driver_ready = icm_tr_driver_ready;
2525		icm->device_connected = icm_tr_device_connected;
2526		icm->device_disconnected = icm_tr_device_disconnected;
2527		icm->xdomain_connected = icm_tr_xdomain_connected;
2528		icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2529		tb->cm_ops = &icm_tr_ops;
2530		break;
2531
2532	case PCI_DEVICE_ID_INTEL_ICL_NHI0:
2533	case PCI_DEVICE_ID_INTEL_ICL_NHI1:
2534		icm->is_supported = icm_fr_is_supported;
2535		icm->driver_ready = icm_icl_driver_ready;
2536		icm->set_uuid = icm_icl_set_uuid;
2537		icm->device_connected = icm_icl_device_connected;
2538		icm->device_disconnected = icm_tr_device_disconnected;
2539		icm->xdomain_connected = icm_tr_xdomain_connected;
2540		icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2541		icm->rtd3_veto = icm_icl_rtd3_veto;
2542		tb->cm_ops = &icm_icl_ops;
2543		break;
2544
2545	case PCI_DEVICE_ID_INTEL_TGL_NHI0:
2546	case PCI_DEVICE_ID_INTEL_TGL_NHI1:
2547	case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:
2548	case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:
2549	case PCI_DEVICE_ID_INTEL_ADL_NHI0:
2550	case PCI_DEVICE_ID_INTEL_ADL_NHI1:
2551	case PCI_DEVICE_ID_INTEL_RPL_NHI0:
2552	case PCI_DEVICE_ID_INTEL_RPL_NHI1:
2553	case PCI_DEVICE_ID_INTEL_MTL_M_NHI0:
2554	case PCI_DEVICE_ID_INTEL_MTL_P_NHI0:
2555	case PCI_DEVICE_ID_INTEL_MTL_P_NHI1:
2556		icm->is_supported = icm_tgl_is_supported;
2557		icm->driver_ready = icm_icl_driver_ready;
2558		icm->set_uuid = icm_icl_set_uuid;
2559		icm->device_connected = icm_icl_device_connected;
2560		icm->device_disconnected = icm_tr_device_disconnected;
2561		icm->xdomain_connected = icm_tr_xdomain_connected;
2562		icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2563		icm->rtd3_veto = icm_icl_rtd3_veto;
2564		tb->cm_ops = &icm_icl_ops;
2565		break;
2566
2567	case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI:
2568	case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:
2569		icm->can_upgrade_nvm = true;
2570		icm->is_supported = icm_tgl_is_supported;
2571		icm->get_mode = icm_ar_get_mode;
2572		icm->driver_ready = icm_tr_driver_ready;
2573		icm->device_connected = icm_tr_device_connected;
2574		icm->device_disconnected = icm_tr_device_disconnected;
2575		icm->xdomain_connected = icm_tr_xdomain_connected;
2576		icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2577		tb->cm_ops = &icm_tr_ops;
2578		break;
2579	}
2580
2581	if (!icm->is_supported || !icm->is_supported(tb)) {
2582		dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n");
2583		tb_domain_put(tb);
2584		return NULL;
2585	}
2586
2587	tb_dbg(tb, "using firmware connection manager\n");
2588
2589	return tb;
2590}