kfd_topology.c

   1// SPDX-License-Identifier: GPL-2.0 OR MIT
   2/*
   3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the "Software"),
   7 * to deal in the Software without restriction, including without limitation
   8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 * and/or sell copies of the Software, and to permit persons to whom the
  10 * Software is furnished to do so, subject to the following conditions:
  11 *
  12 * The above copyright notice and this permission notice shall be included in
  13 * all copies or substantial portions of the Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21 * OTHER DEALINGS IN THE SOFTWARE.
  22 */
  23
  24#include <linux/types.h>
  25#include <linux/kernel.h>
  26#include <linux/pci.h>
  27#include <linux/errno.h>
  28#include <linux/acpi.h>
  29#include <linux/hash.h>
  30#include <linux/cpufreq.h>
  31#include <linux/log2.h>
  32#include <linux/dmi.h>
  33#include <linux/atomic.h>
  34#include <linux/crc16.h>
  35
  36#include "kfd_priv.h"
  37#include "kfd_crat.h"
  38#include "kfd_topology.h"
  39#include "kfd_device_queue_manager.h"
  40#include "kfd_svm.h"
  41#include "kfd_debug.h"
  42#include "amdgpu_amdkfd.h"
  43#include "amdgpu_ras.h"
  44#include "amdgpu.h"
  45
  46/* topology_device_list - Master list of all topology devices */
  47static struct list_head topology_device_list;
  48static struct kfd_system_properties sys_props;
  49
  50static DECLARE_RWSEM(topology_lock);
  51static uint32_t topology_crat_proximity_domain;
  52
  53struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock(
  54						uint32_t proximity_domain)
  55{
  56	struct kfd_topology_device *top_dev;
  57	struct kfd_topology_device *device = NULL;
  58
  59	list_for_each_entry(top_dev, &topology_device_list, list)
  60		if (top_dev->proximity_domain == proximity_domain) {
  61			device = top_dev;
  62			break;
  63		}
  64
  65	return device;
  66}
  67
  68struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
  69						uint32_t proximity_domain)
  70{
  71	struct kfd_topology_device *device = NULL;
  72
  73	down_read(&topology_lock);
  74
  75	device = kfd_topology_device_by_proximity_domain_no_lock(
  76							proximity_domain);
  77	up_read(&topology_lock);
  78
  79	return device;
  80}
  81
  82struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id)
  83{
  84	struct kfd_topology_device *top_dev = NULL;
  85	struct kfd_topology_device *ret = NULL;
  86
  87	down_read(&topology_lock);
  88
  89	list_for_each_entry(top_dev, &topology_device_list, list)
  90		if (top_dev->gpu_id == gpu_id) {
  91			ret = top_dev;
  92			break;
  93		}
  94
  95	up_read(&topology_lock);
  96
  97	return ret;
  98}
  99
 100struct kfd_node *kfd_device_by_id(uint32_t gpu_id)
 101{
 102	struct kfd_topology_device *top_dev;
 103
 104	top_dev = kfd_topology_device_by_id(gpu_id);
 105	if (!top_dev)
 106		return NULL;
 107
 108	return top_dev->gpu;
 109}
 110
 111/* Called with write topology_lock acquired */
 112static void kfd_release_topology_device(struct kfd_topology_device *dev)
 113{
 114	struct kfd_mem_properties *mem;
 115	struct kfd_cache_properties *cache;
 116	struct kfd_iolink_properties *iolink;
 117	struct kfd_iolink_properties *p2plink;
 118	struct kfd_perf_properties *perf;
 119
 120	list_del(&dev->list);
 121
 122	while (dev->mem_props.next != &dev->mem_props) {
 123		mem = container_of(dev->mem_props.next,
 124				struct kfd_mem_properties, list);
 125		list_del(&mem->list);
 126		kfree(mem);
 127	}
 128
 129	while (dev->cache_props.next != &dev->cache_props) {
 130		cache = container_of(dev->cache_props.next,
 131				struct kfd_cache_properties, list);
 132		list_del(&cache->list);
 133		kfree(cache);
 134	}
 135
 136	while (dev->io_link_props.next != &dev->io_link_props) {
 137		iolink = container_of(dev->io_link_props.next,
 138				struct kfd_iolink_properties, list);
 139		list_del(&iolink->list);
 140		kfree(iolink);
 141	}
 142
 143	while (dev->p2p_link_props.next != &dev->p2p_link_props) {
 144		p2plink = container_of(dev->p2p_link_props.next,
 145				struct kfd_iolink_properties, list);
 146		list_del(&p2plink->list);
 147		kfree(p2plink);
 148	}
 149
 150	while (dev->perf_props.next != &dev->perf_props) {
 151		perf = container_of(dev->perf_props.next,
 152				struct kfd_perf_properties, list);
 153		list_del(&perf->list);
 154		kfree(perf);
 155	}
 156
 157	kfree(dev);
 158}
 159
 160void kfd_release_topology_device_list(struct list_head *device_list)
 161{
 162	struct kfd_topology_device *dev;
 163
 164	while (!list_empty(device_list)) {
 165		dev = list_first_entry(device_list,
 166				       struct kfd_topology_device, list);
 167		kfd_release_topology_device(dev);
 168	}
 169}
 170
 171static void kfd_release_live_view(void)
 172{
 173	kfd_release_topology_device_list(&topology_device_list);
 174	memset(&sys_props, 0, sizeof(sys_props));
 175}
 176
 177struct kfd_topology_device *kfd_create_topology_device(
 178				struct list_head *device_list)
 179{
 180	struct kfd_topology_device *dev;
 181
 182	dev = kfd_alloc_struct(dev);
 183	if (!dev) {
 184		pr_err("No memory to allocate a topology device");
 185		return NULL;
 186	}
 187
 188	INIT_LIST_HEAD(&dev->mem_props);
 189	INIT_LIST_HEAD(&dev->cache_props);
 190	INIT_LIST_HEAD(&dev->io_link_props);
 191	INIT_LIST_HEAD(&dev->p2p_link_props);
 192	INIT_LIST_HEAD(&dev->perf_props);
 193
 194	list_add_tail(&dev->list, device_list);
 195
 196	return dev;
 197}
 198
 199
 200#define sysfs_show_gen_prop(buffer, offs, fmt, ...)		\
 201		(offs += snprintf(buffer+offs, PAGE_SIZE-offs,	\
 202				  fmt, __VA_ARGS__))
 203#define sysfs_show_32bit_prop(buffer, offs, name, value) \
 204		sysfs_show_gen_prop(buffer, offs, "%s %u\n", name, value)
 205#define sysfs_show_64bit_prop(buffer, offs, name, value) \
 206		sysfs_show_gen_prop(buffer, offs, "%s %llu\n", name, value)
 207#define sysfs_show_32bit_val(buffer, offs, value) \
 208		sysfs_show_gen_prop(buffer, offs, "%u\n", value)
 209#define sysfs_show_str_val(buffer, offs, value) \
 210		sysfs_show_gen_prop(buffer, offs, "%s\n", value)
 211
 212static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr,
 213		char *buffer)
 214{
 215	int offs = 0;
 216
 217	/* Making sure that the buffer is an empty string */
 218	buffer[0] = 0;
 219
 220	if (attr == &sys_props.attr_genid) {
 221		sysfs_show_32bit_val(buffer, offs,
 222				     sys_props.generation_count);
 223	} else if (attr == &sys_props.attr_props) {
 224		sysfs_show_64bit_prop(buffer, offs, "platform_oem",
 225				      sys_props.platform_oem);
 226		sysfs_show_64bit_prop(buffer, offs, "platform_id",
 227				      sys_props.platform_id);
 228		sysfs_show_64bit_prop(buffer, offs, "platform_rev",
 229				      sys_props.platform_rev);
 230	} else {
 231		offs = -EINVAL;
 232	}
 233
 234	return offs;
 235}
 236
 237static void kfd_topology_kobj_release(struct kobject *kobj)
 238{
 239	kfree(kobj);
 240}
 241
 242static const struct sysfs_ops sysprops_ops = {
 243	.show = sysprops_show,
 244};
 245
 246static const struct kobj_type sysprops_type = {
 247	.release = kfd_topology_kobj_release,
 248	.sysfs_ops = &sysprops_ops,
 249};
 250
 251static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr,
 252		char *buffer)
 253{
 254	int offs = 0;
 255	struct kfd_iolink_properties *iolink;
 256
 257	/* Making sure that the buffer is an empty string */
 258	buffer[0] = 0;
 259
 260	iolink = container_of(attr, struct kfd_iolink_properties, attr);
 261	if (iolink->gpu && kfd_devcgroup_check_permission(iolink->gpu))
 262		return -EPERM;
 263	sysfs_show_32bit_prop(buffer, offs, "type", iolink->iolink_type);
 264	sysfs_show_32bit_prop(buffer, offs, "version_major", iolink->ver_maj);
 265	sysfs_show_32bit_prop(buffer, offs, "version_minor", iolink->ver_min);
 266	sysfs_show_32bit_prop(buffer, offs, "node_from", iolink->node_from);
 267	sysfs_show_32bit_prop(buffer, offs, "node_to", iolink->node_to);
 268	sysfs_show_32bit_prop(buffer, offs, "weight", iolink->weight);
 269	sysfs_show_32bit_prop(buffer, offs, "min_latency", iolink->min_latency);
 270	sysfs_show_32bit_prop(buffer, offs, "max_latency", iolink->max_latency);
 271	sysfs_show_32bit_prop(buffer, offs, "min_bandwidth",
 272			      iolink->min_bandwidth);
 273	sysfs_show_32bit_prop(buffer, offs, "max_bandwidth",
 274			      iolink->max_bandwidth);
 275	sysfs_show_32bit_prop(buffer, offs, "recommended_transfer_size",
 276			      iolink->rec_transfer_size);
 277	sysfs_show_32bit_prop(buffer, offs, "recommended_sdma_engine_id_mask",
 278			      iolink->rec_sdma_eng_id_mask);
 279	sysfs_show_32bit_prop(buffer, offs, "flags", iolink->flags);
 280
 281	return offs;
 282}
 283
 284static const struct sysfs_ops iolink_ops = {
 285	.show = iolink_show,
 286};
 287
 288static const struct kobj_type iolink_type = {
 289	.release = kfd_topology_kobj_release,
 290	.sysfs_ops = &iolink_ops,
 291};
 292
 293static ssize_t mem_show(struct kobject *kobj, struct attribute *attr,
 294		char *buffer)
 295{
 296	int offs = 0;
 297	struct kfd_mem_properties *mem;
 298
 299	/* Making sure that the buffer is an empty string */
 300	buffer[0] = 0;
 301
 302	mem = container_of(attr, struct kfd_mem_properties, attr);
 303	if (mem->gpu && kfd_devcgroup_check_permission(mem->gpu))
 304		return -EPERM;
 305	sysfs_show_32bit_prop(buffer, offs, "heap_type", mem->heap_type);
 306	sysfs_show_64bit_prop(buffer, offs, "size_in_bytes",
 307			      mem->size_in_bytes);
 308	sysfs_show_32bit_prop(buffer, offs, "flags", mem->flags);
 309	sysfs_show_32bit_prop(buffer, offs, "width", mem->width);
 310	sysfs_show_32bit_prop(buffer, offs, "mem_clk_max",
 311			      mem->mem_clk_max);
 312
 313	return offs;
 314}
 315
 316static const struct sysfs_ops mem_ops = {
 317	.show = mem_show,
 318};
 319
 320static const struct kobj_type mem_type = {
 321	.release = kfd_topology_kobj_release,
 322	.sysfs_ops = &mem_ops,
 323};
 324
 325static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
 326		char *buffer)
 327{
 328	int offs = 0;
 329	uint32_t i, j;
 330	struct kfd_cache_properties *cache;
 331
 332	/* Making sure that the buffer is an empty string */
 333	buffer[0] = 0;
 334	cache = container_of(attr, struct kfd_cache_properties, attr);
 335	if (cache->gpu && kfd_devcgroup_check_permission(cache->gpu))
 336		return -EPERM;
 337	sysfs_show_32bit_prop(buffer, offs, "processor_id_low",
 338			cache->processor_id_low);
 339	sysfs_show_32bit_prop(buffer, offs, "level", cache->cache_level);
 340	sysfs_show_32bit_prop(buffer, offs, "size", cache->cache_size);
 341	sysfs_show_32bit_prop(buffer, offs, "cache_line_size",
 342			      cache->cacheline_size);
 343	sysfs_show_32bit_prop(buffer, offs, "cache_lines_per_tag",
 344			      cache->cachelines_per_tag);
 345	sysfs_show_32bit_prop(buffer, offs, "association", cache->cache_assoc);
 346	sysfs_show_32bit_prop(buffer, offs, "latency", cache->cache_latency);
 347	sysfs_show_32bit_prop(buffer, offs, "type", cache->cache_type);
 348
 349	offs += snprintf(buffer+offs, PAGE_SIZE-offs, "sibling_map ");
 350	for (i = 0; i < cache->sibling_map_size; i++)
 351		for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++)
 352			/* Check each bit */
 353			offs += snprintf(buffer+offs, PAGE_SIZE-offs, "%d,",
 354						(cache->sibling_map[i] >> j) & 1);
 355
 356	/* Replace the last "," with end of line */
 357	buffer[offs-1] = '\n';
 358	return offs;
 359}
 360
 361static const struct sysfs_ops cache_ops = {
 362	.show = kfd_cache_show,
 363};
 364
 365static const struct kobj_type cache_type = {
 366	.release = kfd_topology_kobj_release,
 367	.sysfs_ops = &cache_ops,
 368};
 369
 370/****** Sysfs of Performance Counters ******/
 371
 372struct kfd_perf_attr {
 373	struct kobj_attribute attr;
 374	uint32_t data;
 375};
 376
 377static ssize_t perf_show(struct kobject *kobj, struct kobj_attribute *attrs,
 378			char *buf)
 379{
 380	int offs = 0;
 381	struct kfd_perf_attr *attr;
 382
 383	buf[0] = 0;
 384	attr = container_of(attrs, struct kfd_perf_attr, attr);
 385	if (!attr->data) /* invalid data for PMC */
 386		return 0;
 387	else
 388		return sysfs_show_32bit_val(buf, offs, attr->data);
 389}
 390
 391#define KFD_PERF_DESC(_name, _data)			\
 392{							\
 393	.attr  = __ATTR(_name, 0444, perf_show, NULL),	\
 394	.data = _data,					\
 395}
 396
 397static struct kfd_perf_attr perf_attr_iommu[] = {
 398	KFD_PERF_DESC(max_concurrent, 0),
 399	KFD_PERF_DESC(num_counters, 0),
 400	KFD_PERF_DESC(counter_ids, 0),
 401};
 402/****************************************/
 403
 404static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 405		char *buffer)
 406{
 407	int offs = 0;
 408	struct kfd_topology_device *dev;
 409	uint32_t log_max_watch_addr;
 410
 411	/* Making sure that the buffer is an empty string */
 412	buffer[0] = 0;
 413
 414	if (strcmp(attr->name, "gpu_id") == 0) {
 415		dev = container_of(attr, struct kfd_topology_device,
 416				attr_gpuid);
 417		if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
 418			return -EPERM;
 419		return sysfs_show_32bit_val(buffer, offs, dev->gpu_id);
 420	}
 421
 422	if (strcmp(attr->name, "name") == 0) {
 423		dev = container_of(attr, struct kfd_topology_device,
 424				attr_name);
 425
 426		if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
 427			return -EPERM;
 428		return sysfs_show_str_val(buffer, offs, dev->node_props.name);
 429	}
 430
 431	dev = container_of(attr, struct kfd_topology_device,
 432			attr_props);
 433	if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu))
 434		return -EPERM;
 435	sysfs_show_32bit_prop(buffer, offs, "cpu_cores_count",
 436			      dev->node_props.cpu_cores_count);
 437	sysfs_show_32bit_prop(buffer, offs, "simd_count",
 438			      dev->gpu ? dev->node_props.simd_count : 0);
 439	sysfs_show_32bit_prop(buffer, offs, "mem_banks_count",
 440			      dev->node_props.mem_banks_count);
 441	sysfs_show_32bit_prop(buffer, offs, "caches_count",
 442			      dev->node_props.caches_count);
 443	sysfs_show_32bit_prop(buffer, offs, "io_links_count",
 444			      dev->node_props.io_links_count);
 445	sysfs_show_32bit_prop(buffer, offs, "p2p_links_count",
 446			      dev->node_props.p2p_links_count);
 447	sysfs_show_32bit_prop(buffer, offs, "cpu_core_id_base",
 448			      dev->node_props.cpu_core_id_base);
 449	sysfs_show_32bit_prop(buffer, offs, "simd_id_base",
 450			      dev->node_props.simd_id_base);
 451	sysfs_show_32bit_prop(buffer, offs, "max_waves_per_simd",
 452			      dev->node_props.max_waves_per_simd);
 453	sysfs_show_32bit_prop(buffer, offs, "lds_size_in_kb",
 454			      dev->node_props.lds_size_in_kb);
 455	sysfs_show_32bit_prop(buffer, offs, "gds_size_in_kb",
 456			      dev->node_props.gds_size_in_kb);
 457	sysfs_show_32bit_prop(buffer, offs, "num_gws",
 458			      dev->node_props.num_gws);
 459	sysfs_show_32bit_prop(buffer, offs, "wave_front_size",
 460			      dev->node_props.wave_front_size);
 461	sysfs_show_32bit_prop(buffer, offs, "array_count",
 462			      dev->gpu ? (dev->node_props.array_count *
 463					  NUM_XCC(dev->gpu->xcc_mask)) : 0);
 464	sysfs_show_32bit_prop(buffer, offs, "simd_arrays_per_engine",
 465			      dev->node_props.simd_arrays_per_engine);
 466	sysfs_show_32bit_prop(buffer, offs, "cu_per_simd_array",
 467			      dev->node_props.cu_per_simd_array);
 468	sysfs_show_32bit_prop(buffer, offs, "simd_per_cu",
 469			      dev->node_props.simd_per_cu);
 470	sysfs_show_32bit_prop(buffer, offs, "max_slots_scratch_cu",
 471			      dev->node_props.max_slots_scratch_cu);
 472	sysfs_show_32bit_prop(buffer, offs, "gfx_target_version",
 473			      dev->node_props.gfx_target_version);
 474	sysfs_show_32bit_prop(buffer, offs, "vendor_id",
 475			      dev->node_props.vendor_id);
 476	sysfs_show_32bit_prop(buffer, offs, "device_id",
 477			      dev->node_props.device_id);
 478	sysfs_show_32bit_prop(buffer, offs, "location_id",
 479			      dev->node_props.location_id);
 480	sysfs_show_32bit_prop(buffer, offs, "domain",
 481			      dev->node_props.domain);
 482	sysfs_show_32bit_prop(buffer, offs, "drm_render_minor",
 483			      dev->node_props.drm_render_minor);
 484	sysfs_show_64bit_prop(buffer, offs, "hive_id",
 485			      dev->node_props.hive_id);
 486	sysfs_show_32bit_prop(buffer, offs, "num_sdma_engines",
 487			      dev->node_props.num_sdma_engines);
 488	sysfs_show_32bit_prop(buffer, offs, "num_sdma_xgmi_engines",
 489			      dev->node_props.num_sdma_xgmi_engines);
 490	sysfs_show_32bit_prop(buffer, offs, "num_sdma_queues_per_engine",
 491			      dev->node_props.num_sdma_queues_per_engine);
 492	sysfs_show_32bit_prop(buffer, offs, "num_cp_queues",
 493			      dev->node_props.num_cp_queues);
 494	sysfs_show_32bit_prop(buffer, offs, "cwsr_size",
 495			      dev->node_props.cwsr_size);
 496	sysfs_show_32bit_prop(buffer, offs, "ctl_stack_size",
 497			      dev->node_props.ctl_stack_size);
 498
 499	if (dev->gpu) {
 500		log_max_watch_addr =
 501			__ilog2_u32(dev->gpu->kfd->device_info.num_of_watch_points);
 502
 503		if (log_max_watch_addr) {
 504			dev->node_props.capability |=
 505					HSA_CAP_WATCH_POINTS_SUPPORTED;
 506
 507			dev->node_props.capability |=
 508				((log_max_watch_addr <<
 509					HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT) &
 510				HSA_CAP_WATCH_POINTS_TOTALBITS_MASK);
 511		}
 512
 513		if (dev->gpu->adev->asic_type == CHIP_TONGA)
 514			dev->node_props.capability |=
 515					HSA_CAP_AQL_QUEUE_DOUBLE_MAP;
 516
 517		if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0) &&
 518			(dev->gpu->adev->sdma.supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE))
 519				dev->node_props.capability2 |= HSA_CAP2_PER_SDMA_QUEUE_RESET_SUPPORTED;
 520
 521		sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_fcompute",
 522			dev->node_props.max_engine_clk_fcompute);
 523
 524		sysfs_show_64bit_prop(buffer, offs, "local_mem_size", 0ULL);
 525
 526		sysfs_show_32bit_prop(buffer, offs, "fw_version",
 527				      dev->gpu->kfd->mec_fw_version);
 528		sysfs_show_32bit_prop(buffer, offs, "capability",
 529				      dev->node_props.capability);
 530		sysfs_show_32bit_prop(buffer, offs, "capability2",
 531				      dev->node_props.capability2);
 532		sysfs_show_64bit_prop(buffer, offs, "debug_prop",
 533				      dev->node_props.debug_prop);
 534		sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version",
 535				      dev->gpu->kfd->sdma_fw_version);
 536		sysfs_show_64bit_prop(buffer, offs, "unique_id",
 537				      dev->gpu->xcp &&
 538				      (dev->gpu->xcp->xcp_mgr->mode !=
 539				       AMDGPU_SPX_PARTITION_MODE) ?
 540				      dev->gpu->xcp->unique_id :
 541				      dev->gpu->adev->unique_id);
 542		sysfs_show_32bit_prop(buffer, offs, "num_xcc",
 543				      NUM_XCC(dev->gpu->xcc_mask));
 544	}
 545
 546	return sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_ccompute",
 547				     cpufreq_quick_get_max(0)/1000);
 548}
 549
 550static const struct sysfs_ops node_ops = {
 551	.show = node_show,
 552};
 553
 554static const struct kobj_type node_type = {
 555	.release = kfd_topology_kobj_release,
 556	.sysfs_ops = &node_ops,
 557};
 558
 559static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr)
 560{
 561	sysfs_remove_file(kobj, attr);
 562	kobject_del(kobj);
 563	kobject_put(kobj);
 564}
 565
 566static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
 567{
 568	struct kfd_iolink_properties *p2plink;
 569	struct kfd_iolink_properties *iolink;
 570	struct kfd_cache_properties *cache;
 571	struct kfd_mem_properties *mem;
 572	struct kfd_perf_properties *perf;
 573
 574	if (dev->kobj_iolink) {
 575		list_for_each_entry(iolink, &dev->io_link_props, list)
 576			if (iolink->kobj) {
 577				kfd_remove_sysfs_file(iolink->kobj,
 578							&iolink->attr);
 579				iolink->kobj = NULL;
 580			}
 581		kobject_del(dev->kobj_iolink);
 582		kobject_put(dev->kobj_iolink);
 583		dev->kobj_iolink = NULL;
 584	}
 585
 586	if (dev->kobj_p2plink) {
 587		list_for_each_entry(p2plink, &dev->p2p_link_props, list)
 588			if (p2plink->kobj) {
 589				kfd_remove_sysfs_file(p2plink->kobj,
 590							&p2plink->attr);
 591				p2plink->kobj = NULL;
 592			}
 593		kobject_del(dev->kobj_p2plink);
 594		kobject_put(dev->kobj_p2plink);
 595		dev->kobj_p2plink = NULL;
 596	}
 597
 598	if (dev->kobj_cache) {
 599		list_for_each_entry(cache, &dev->cache_props, list)
 600			if (cache->kobj) {
 601				kfd_remove_sysfs_file(cache->kobj,
 602							&cache->attr);
 603				cache->kobj = NULL;
 604			}
 605		kobject_del(dev->kobj_cache);
 606		kobject_put(dev->kobj_cache);
 607		dev->kobj_cache = NULL;
 608	}
 609
 610	if (dev->kobj_mem) {
 611		list_for_each_entry(mem, &dev->mem_props, list)
 612			if (mem->kobj) {
 613				kfd_remove_sysfs_file(mem->kobj, &mem->attr);
 614				mem->kobj = NULL;
 615			}
 616		kobject_del(dev->kobj_mem);
 617		kobject_put(dev->kobj_mem);
 618		dev->kobj_mem = NULL;
 619	}
 620
 621	if (dev->kobj_perf) {
 622		list_for_each_entry(perf, &dev->perf_props, list) {
 623			kfree(perf->attr_group);
 624			perf->attr_group = NULL;
 625		}
 626		kobject_del(dev->kobj_perf);
 627		kobject_put(dev->kobj_perf);
 628		dev->kobj_perf = NULL;
 629	}
 630
 631	if (dev->kobj_node) {
 632		sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid);
 633		sysfs_remove_file(dev->kobj_node, &dev->attr_name);
 634		sysfs_remove_file(dev->kobj_node, &dev->attr_props);
 635		kobject_del(dev->kobj_node);
 636		kobject_put(dev->kobj_node);
 637		dev->kobj_node = NULL;
 638	}
 639}
 640
 641static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
 642		uint32_t id)
 643{
 644	struct kfd_iolink_properties *p2plink;
 645	struct kfd_iolink_properties *iolink;
 646	struct kfd_cache_properties *cache;
 647	struct kfd_mem_properties *mem;
 648	struct kfd_perf_properties *perf;
 649	int ret;
 650	uint32_t i, num_attrs;
 651	struct attribute **attrs;
 652
 653	if (WARN_ON(dev->kobj_node))
 654		return -EEXIST;
 655
 656	/*
 657	 * Creating the sysfs folders
 658	 */
 659	dev->kobj_node = kfd_alloc_struct(dev->kobj_node);
 660	if (!dev->kobj_node)
 661		return -ENOMEM;
 662
 663	ret = kobject_init_and_add(dev->kobj_node, &node_type,
 664			sys_props.kobj_nodes, "%d", id);
 665	if (ret < 0) {
 666		kobject_put(dev->kobj_node);
 667		return ret;
 668	}
 669
 670	dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node);
 671	if (!dev->kobj_mem)
 672		return -ENOMEM;
 673
 674	dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node);
 675	if (!dev->kobj_cache)
 676		return -ENOMEM;
 677
 678	dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node);
 679	if (!dev->kobj_iolink)
 680		return -ENOMEM;
 681
 682	dev->kobj_p2plink = kobject_create_and_add("p2p_links", dev->kobj_node);
 683	if (!dev->kobj_p2plink)
 684		return -ENOMEM;
 685
 686	dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node);
 687	if (!dev->kobj_perf)
 688		return -ENOMEM;
 689
 690	/*
 691	 * Creating sysfs files for node properties
 692	 */
 693	dev->attr_gpuid.name = "gpu_id";
 694	dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE;
 695	sysfs_attr_init(&dev->attr_gpuid);
 696	dev->attr_name.name = "name";
 697	dev->attr_name.mode = KFD_SYSFS_FILE_MODE;
 698	sysfs_attr_init(&dev->attr_name);
 699	dev->attr_props.name = "properties";
 700	dev->attr_props.mode = KFD_SYSFS_FILE_MODE;
 701	sysfs_attr_init(&dev->attr_props);
 702	ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid);
 703	if (ret < 0)
 704		return ret;
 705	ret = sysfs_create_file(dev->kobj_node, &dev->attr_name);
 706	if (ret < 0)
 707		return ret;
 708	ret = sysfs_create_file(dev->kobj_node, &dev->attr_props);
 709	if (ret < 0)
 710		return ret;
 711
 712	i = 0;
 713	list_for_each_entry(mem, &dev->mem_props, list) {
 714		mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 715		if (!mem->kobj)
 716			return -ENOMEM;
 717		ret = kobject_init_and_add(mem->kobj, &mem_type,
 718				dev->kobj_mem, "%d", i);
 719		if (ret < 0) {
 720			kobject_put(mem->kobj);
 721			return ret;
 722		}
 723
 724		mem->attr.name = "properties";
 725		mem->attr.mode = KFD_SYSFS_FILE_MODE;
 726		sysfs_attr_init(&mem->attr);
 727		ret = sysfs_create_file(mem->kobj, &mem->attr);
 728		if (ret < 0)
 729			return ret;
 730		i++;
 731	}
 732
 733	i = 0;
 734	list_for_each_entry(cache, &dev->cache_props, list) {
 735		cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 736		if (!cache->kobj)
 737			return -ENOMEM;
 738		ret = kobject_init_and_add(cache->kobj, &cache_type,
 739				dev->kobj_cache, "%d", i);
 740		if (ret < 0) {
 741			kobject_put(cache->kobj);
 742			return ret;
 743		}
 744
 745		cache->attr.name = "properties";
 746		cache->attr.mode = KFD_SYSFS_FILE_MODE;
 747		sysfs_attr_init(&cache->attr);
 748		ret = sysfs_create_file(cache->kobj, &cache->attr);
 749		if (ret < 0)
 750			return ret;
 751		i++;
 752	}
 753
 754	i = 0;
 755	list_for_each_entry(iolink, &dev->io_link_props, list) {
 756		iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 757		if (!iolink->kobj)
 758			return -ENOMEM;
 759		ret = kobject_init_and_add(iolink->kobj, &iolink_type,
 760				dev->kobj_iolink, "%d", i);
 761		if (ret < 0) {
 762			kobject_put(iolink->kobj);
 763			return ret;
 764		}
 765
 766		iolink->attr.name = "properties";
 767		iolink->attr.mode = KFD_SYSFS_FILE_MODE;
 768		sysfs_attr_init(&iolink->attr);
 769		ret = sysfs_create_file(iolink->kobj, &iolink->attr);
 770		if (ret < 0)
 771			return ret;
 772		i++;
 773	}
 774
 775	i = 0;
 776	list_for_each_entry(p2plink, &dev->p2p_link_props, list) {
 777		p2plink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
 778		if (!p2plink->kobj)
 779			return -ENOMEM;
 780		ret = kobject_init_and_add(p2plink->kobj, &iolink_type,
 781				dev->kobj_p2plink, "%d", i);
 782		if (ret < 0) {
 783			kobject_put(p2plink->kobj);
 784			return ret;
 785		}
 786
 787		p2plink->attr.name = "properties";
 788		p2plink->attr.mode = KFD_SYSFS_FILE_MODE;
 789		sysfs_attr_init(&p2plink->attr);
 790		ret = sysfs_create_file(p2plink->kobj, &p2plink->attr);
 791		if (ret < 0)
 792			return ret;
 793		i++;
 794	}
 795
 796	/* All hardware blocks have the same number of attributes. */
 797	num_attrs = ARRAY_SIZE(perf_attr_iommu);
 798	list_for_each_entry(perf, &dev->perf_props, list) {
 799		perf->attr_group = kzalloc(sizeof(struct kfd_perf_attr)
 800			* num_attrs + sizeof(struct attribute_group),
 801			GFP_KERNEL);
 802		if (!perf->attr_group)
 803			return -ENOMEM;
 804
 805		attrs = (struct attribute **)(perf->attr_group + 1);
 806		if (!strcmp(perf->block_name, "iommu")) {
 807		/* Information of IOMMU's num_counters and counter_ids is shown
 808		 * under /sys/bus/event_source/devices/amd_iommu. We don't
 809		 * duplicate here.
 810		 */
 811			perf_attr_iommu[0].data = perf->max_concurrent;
 812			for (i = 0; i < num_attrs; i++)
 813				attrs[i] = &perf_attr_iommu[i].attr.attr;
 814		}
 815		perf->attr_group->name = perf->block_name;
 816		perf->attr_group->attrs = attrs;
 817		ret = sysfs_create_group(dev->kobj_perf, perf->attr_group);
 818		if (ret < 0)
 819			return ret;
 820	}
 821
 822	return 0;
 823}
 824
 825/* Called with write topology lock acquired */
 826static int kfd_build_sysfs_node_tree(void)
 827{
 828	struct kfd_topology_device *dev;
 829	int ret;
 830	uint32_t i = 0;
 831
 832	list_for_each_entry(dev, &topology_device_list, list) {
 833		ret = kfd_build_sysfs_node_entry(dev, i);
 834		if (ret < 0)
 835			return ret;
 836		i++;
 837	}
 838
 839	return 0;
 840}
 841
 842/* Called with write topology lock acquired */
 843static void kfd_remove_sysfs_node_tree(void)
 844{
 845	struct kfd_topology_device *dev;
 846
 847	list_for_each_entry(dev, &topology_device_list, list)
 848		kfd_remove_sysfs_node_entry(dev);
 849}
 850
 851static int kfd_topology_update_sysfs(void)
 852{
 853	int ret;
 854
 855	if (!sys_props.kobj_topology) {
 856		sys_props.kobj_topology =
 857				kfd_alloc_struct(sys_props.kobj_topology);
 858		if (!sys_props.kobj_topology)
 859			return -ENOMEM;
 860
 861		ret = kobject_init_and_add(sys_props.kobj_topology,
 862				&sysprops_type,  &kfd_device->kobj,
 863				"topology");
 864		if (ret < 0) {
 865			kobject_put(sys_props.kobj_topology);
 866			return ret;
 867		}
 868
 869		sys_props.kobj_nodes = kobject_create_and_add("nodes",
 870				sys_props.kobj_topology);
 871		if (!sys_props.kobj_nodes)
 872			return -ENOMEM;
 873
 874		sys_props.attr_genid.name = "generation_id";
 875		sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE;
 876		sysfs_attr_init(&sys_props.attr_genid);
 877		ret = sysfs_create_file(sys_props.kobj_topology,
 878				&sys_props.attr_genid);
 879		if (ret < 0)
 880			return ret;
 881
 882		sys_props.attr_props.name = "system_properties";
 883		sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE;
 884		sysfs_attr_init(&sys_props.attr_props);
 885		ret = sysfs_create_file(sys_props.kobj_topology,
 886				&sys_props.attr_props);
 887		if (ret < 0)
 888			return ret;
 889	}
 890
 891	kfd_remove_sysfs_node_tree();
 892
 893	return kfd_build_sysfs_node_tree();
 894}
 895
 896static void kfd_topology_release_sysfs(void)
 897{
 898	kfd_remove_sysfs_node_tree();
 899	if (sys_props.kobj_topology) {
 900		sysfs_remove_file(sys_props.kobj_topology,
 901				&sys_props.attr_genid);
 902		sysfs_remove_file(sys_props.kobj_topology,
 903				&sys_props.attr_props);
 904		if (sys_props.kobj_nodes) {
 905			kobject_del(sys_props.kobj_nodes);
 906			kobject_put(sys_props.kobj_nodes);
 907			sys_props.kobj_nodes = NULL;
 908		}
 909		kobject_del(sys_props.kobj_topology);
 910		kobject_put(sys_props.kobj_topology);
 911		sys_props.kobj_topology = NULL;
 912	}
 913}
 914
 915/* Called with write topology_lock acquired */
 916static void kfd_topology_update_device_list(struct list_head *temp_list,
 917					struct list_head *master_list)
 918{
 919	while (!list_empty(temp_list)) {
 920		list_move_tail(temp_list->next, master_list);
 921		sys_props.num_devices++;
 922	}
 923}
 924
 925static void kfd_debug_print_topology(void)
 926{
 927	struct kfd_topology_device *dev;
 928
 929	down_read(&topology_lock);
 930
 931	dev = list_last_entry(&topology_device_list,
 932			struct kfd_topology_device, list);
 933	if (dev) {
 934		if (dev->node_props.cpu_cores_count &&
 935				dev->node_props.simd_count) {
 936			pr_info("Topology: Add APU node [0x%0x:0x%0x]\n",
 937				dev->node_props.device_id,
 938				dev->node_props.vendor_id);
 939		} else if (dev->node_props.cpu_cores_count)
 940			pr_info("Topology: Add CPU node\n");
 941		else if (dev->node_props.simd_count)
 942			pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n",
 943				dev->node_props.device_id,
 944				dev->node_props.vendor_id);
 945	}
 946	up_read(&topology_lock);
 947}
 948
 949/* Helper function for intializing platform_xx members of
 950 * kfd_system_properties. Uses OEM info from the last CPU/APU node.
 951 */
 952static void kfd_update_system_properties(void)
 953{
 954	struct kfd_topology_device *dev;
 955
 956	down_read(&topology_lock);
 957	dev = list_last_entry(&topology_device_list,
 958			struct kfd_topology_device, list);
 959	if (dev) {
 960		sys_props.platform_id = dev->oem_id64;
 961		sys_props.platform_oem = *((uint64_t *)dev->oem_table_id);
 962		sys_props.platform_rev = dev->oem_revision;
 963	}
 964	up_read(&topology_lock);
 965}
 966
 967static void find_system_memory(const struct dmi_header *dm, void *private)
 968{
 969	struct dmi_mem_device *memdev = container_of(dm, struct dmi_mem_device, header);
 970	struct kfd_mem_properties *mem;
 971	struct kfd_topology_device *kdev =
 972		(struct kfd_topology_device *)private;
 973
 974	if (memdev->header.type != DMI_ENTRY_MEM_DEVICE)
 975		return;
 976	if (memdev->header.length < sizeof(struct dmi_mem_device))
 977		return;
 978
 979	list_for_each_entry(mem, &kdev->mem_props, list) {
 980		if (memdev->total_width != 0xFFFF && memdev->total_width != 0)
 981			mem->width = memdev->total_width;
 982		if (memdev->speed != 0)
 983			mem->mem_clk_max = memdev->speed;
 984	}
 985}
 986
 987/* kfd_add_non_crat_information - Add information that is not currently
 988 *	defined in CRAT but is necessary for KFD topology
 989 * @dev - topology device to which addition info is added
 990 */
 991static void kfd_add_non_crat_information(struct kfd_topology_device *kdev)
 992{
 993	/* Check if CPU only node. */
 994	if (!kdev->gpu) {
 995		/* Add system memory information */
 996		dmi_walk(find_system_memory, kdev);
 997	}
 998	/* TODO: For GPU node, rearrange code from kfd_topology_add_device */
 999}
1000
1001int kfd_topology_init(void)
1002{
1003	void *crat_image = NULL;
1004	size_t image_size = 0;
1005	int ret;
1006	struct list_head temp_topology_device_list;
1007	int cpu_only_node = 0;
1008	struct kfd_topology_device *kdev;
1009	int proximity_domain;
1010
1011	/* topology_device_list - Master list of all topology devices
1012	 * temp_topology_device_list - temporary list created while parsing CRAT
1013	 * or VCRAT. Once parsing is complete the contents of list is moved to
1014	 * topology_device_list
1015	 */
1016
1017	/* Initialize the head for the both the lists */
1018	INIT_LIST_HEAD(&topology_device_list);
1019	INIT_LIST_HEAD(&temp_topology_device_list);
1020	init_rwsem(&topology_lock);
1021
1022	memset(&sys_props, 0, sizeof(sys_props));
1023
1024	/* Proximity domains in ACPI CRAT tables start counting at
1025	 * 0. The same should be true for virtual CRAT tables created
1026	 * at this stage. GPUs added later in kfd_topology_add_device
1027	 * use a counter.
1028	 */
1029	proximity_domain = 0;
1030
1031	ret = kfd_create_crat_image_virtual(&crat_image, &image_size,
1032					    COMPUTE_UNIT_CPU, NULL,
1033					    proximity_domain);
1034	cpu_only_node = 1;
1035	if (ret) {
1036		pr_err("Error creating VCRAT table for CPU\n");
1037		return ret;
1038	}
1039
1040	ret = kfd_parse_crat_table(crat_image,
1041				   &temp_topology_device_list,
1042				   proximity_domain);
1043	if (ret) {
1044		pr_err("Error parsing VCRAT table for CPU\n");
1045		goto err;
1046	}
1047
1048	kdev = list_first_entry(&temp_topology_device_list,
1049				struct kfd_topology_device, list);
1050
1051	down_write(&topology_lock);
1052	kfd_topology_update_device_list(&temp_topology_device_list,
1053					&topology_device_list);
1054	topology_crat_proximity_domain = sys_props.num_devices-1;
1055	ret = kfd_topology_update_sysfs();
1056	up_write(&topology_lock);
1057
1058	if (!ret) {
1059		sys_props.generation_count++;
1060		kfd_update_system_properties();
1061		kfd_debug_print_topology();
1062	} else
1063		pr_err("Failed to update topology in sysfs ret=%d\n", ret);
1064
1065	/* For nodes with GPU, this information gets added
1066	 * when GPU is detected (kfd_topology_add_device).
1067	 */
1068	if (cpu_only_node) {
1069		/* Add additional information to CPU only node created above */
1070		down_write(&topology_lock);
1071		kdev = list_first_entry(&topology_device_list,
1072				struct kfd_topology_device, list);
1073		up_write(&topology_lock);
1074		kfd_add_non_crat_information(kdev);
1075	}
1076
1077err:
1078	kfd_destroy_crat_image(crat_image);
1079	return ret;
1080}
1081
1082void kfd_topology_shutdown(void)
1083{
1084	down_write(&topology_lock);
1085	kfd_topology_release_sysfs();
1086	kfd_release_live_view();
1087	up_write(&topology_lock);
1088}
1089
1090static uint32_t kfd_generate_gpu_id(struct kfd_node *gpu)
1091{
1092	uint32_t gpu_id;
1093	uint32_t buf[8];
1094	uint64_t local_mem_size;
1095	struct kfd_topology_device *dev;
1096	bool is_unique;
1097	uint8_t *crc_buf;
1098
1099	if (!gpu)
1100		return 0;
1101
1102	crc_buf = (uint8_t *)&buf;
1103	local_mem_size = gpu->local_mem_info.local_mem_size_private +
1104			gpu->local_mem_info.local_mem_size_public;
1105	buf[0] = gpu->adev->pdev->devfn;
1106	buf[1] = gpu->adev->pdev->subsystem_vendor |
1107		(gpu->adev->pdev->subsystem_device << 16);
1108	buf[2] = pci_domain_nr(gpu->adev->pdev->bus);
1109	buf[3] = gpu->adev->pdev->device;
1110	buf[4] = gpu->adev->pdev->bus->number;
1111	buf[5] = lower_32_bits(local_mem_size);
1112	buf[6] = upper_32_bits(local_mem_size);
1113	buf[7] = (ffs(gpu->xcc_mask) - 1) | (NUM_XCC(gpu->xcc_mask) << 16);
1114
1115	gpu_id = crc16(0, crc_buf, sizeof(buf)) &
1116		 ((1 << KFD_GPU_ID_HASH_WIDTH) - 1);
1117
1118	/* There is a very small possibility when generating a
1119	 * 16 (KFD_GPU_ID_HASH_WIDTH) bit value from 8 word buffer
1120	 * that the value could be 0 or non-unique. So, check if
1121	 * it is unique and non-zero. If not unique increment till
1122	 * unique one is found. In case of overflow, restart from 1
1123	 */
1124
1125	down_read(&topology_lock);
1126	do {
1127		is_unique = true;
1128		if (!gpu_id)
1129			gpu_id = 1;
1130		list_for_each_entry(dev, &topology_device_list, list) {
1131			if (dev->gpu && dev->gpu_id == gpu_id) {
1132				is_unique = false;
1133				break;
1134			}
1135		}
1136		if (unlikely(!is_unique))
1137			gpu_id = (gpu_id + 1) &
1138				  ((1 << KFD_GPU_ID_HASH_WIDTH) - 1);
1139	} while (!is_unique);
1140	up_read(&topology_lock);
1141
1142	return gpu_id;
1143}
1144/* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If
1145 *		the GPU device is not already present in the topology device
1146 *		list then return NULL. This means a new topology device has to
1147 *		be created for this GPU.
1148 */
1149static struct kfd_topology_device *kfd_assign_gpu(struct kfd_node *gpu)
1150{
1151	struct kfd_topology_device *dev;
1152	struct kfd_topology_device *out_dev = NULL;
1153	struct kfd_mem_properties *mem;
1154	struct kfd_cache_properties *cache;
1155	struct kfd_iolink_properties *iolink;
1156	struct kfd_iolink_properties *p2plink;
1157
1158	list_for_each_entry(dev, &topology_device_list, list) {
1159		/* Discrete GPUs need their own topology device list
1160		 * entries. Don't assign them to CPU/APU nodes.
1161		 */
1162		if (dev->node_props.cpu_cores_count)
1163			continue;
1164
1165		if (!dev->gpu && (dev->node_props.simd_count > 0)) {
1166			dev->gpu = gpu;
1167			out_dev = dev;
1168
1169			list_for_each_entry(mem, &dev->mem_props, list)
1170				mem->gpu = dev->gpu;
1171			list_for_each_entry(cache, &dev->cache_props, list)
1172				cache->gpu = dev->gpu;
1173			list_for_each_entry(iolink, &dev->io_link_props, list)
1174				iolink->gpu = dev->gpu;
1175			list_for_each_entry(p2plink, &dev->p2p_link_props, list)
1176				p2plink->gpu = dev->gpu;
1177			break;
1178		}
1179	}
1180	return out_dev;
1181}
1182
1183static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
1184{
1185	/*
1186	 * TODO: Generate an event for thunk about the arrival/removal
1187	 * of the GPU
1188	 */
1189}
1190
1191/* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info,
1192 *		patch this after CRAT parsing.
1193 */
1194static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
1195{
1196	struct kfd_mem_properties *mem;
1197	struct kfd_local_mem_info local_mem_info;
1198
1199	if (!dev)
1200		return;
1201
1202	/* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with
1203	 * single bank of VRAM local memory.
1204	 * for dGPUs - VCRAT reports only one bank of Local Memory
1205	 * for APUs - If CRAT from ACPI reports more than one bank, then
1206	 *	all the banks will report the same mem_clk_max information
1207	 */
1208	amdgpu_amdkfd_get_local_mem_info(dev->gpu->adev, &local_mem_info,
1209					 dev->gpu->xcp);
1210
1211	list_for_each_entry(mem, &dev->mem_props, list)
1212		mem->mem_clk_max = local_mem_info.mem_clk_max;
1213}
1214
1215static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev,
1216					struct kfd_topology_device *target_gpu_dev,
1217					struct kfd_iolink_properties *link)
1218{
1219	/* xgmi always supports atomics between links. */
1220	if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI)
1221		return;
1222
1223	/* check pcie support to set cpu(dev) flags for target_gpu_dev link. */
1224	if (target_gpu_dev) {
1225		uint32_t cap;
1226
1227		pcie_capability_read_dword(target_gpu_dev->gpu->adev->pdev,
1228				PCI_EXP_DEVCAP2, &cap);
1229
1230		if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
1231			     PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
1232			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
1233				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
1234	/* set gpu (dev) flags. */
1235	} else {
1236		if (!dev->gpu->kfd->pci_atomic_requested ||
1237				dev->gpu->adev->asic_type == CHIP_HAWAII)
1238			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
1239				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
1240	}
1241}
1242
1243static void kfd_set_iolink_non_coherent(struct kfd_topology_device *to_dev,
1244		struct kfd_iolink_properties *outbound_link,
1245		struct kfd_iolink_properties *inbound_link)
1246{
1247	/* CPU -> GPU with PCIe */
1248	if (!to_dev->gpu &&
1249	    inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
1250		inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT;
1251
1252	if (to_dev->gpu) {
1253		/* GPU <-> GPU with PCIe and
1254		 * Vega20 with XGMI
1255		 */
1256		if (inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS ||
1257		    (inbound_link->iolink_type == CRAT_IOLINK_TYPE_XGMI &&
1258		    KFD_GC_VERSION(to_dev->gpu) == IP_VERSION(9, 4, 0))) {
1259			outbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT;
1260			inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT;
1261		}
1262	}
1263}
1264
1265#define REC_SDMA_NUM_GPU	8
1266static const int rec_sdma_eng_map[REC_SDMA_NUM_GPU][REC_SDMA_NUM_GPU] = {
1267							{ -1, 14, 12, 2, 4, 8, 10, 6 },
1268							{ 14, -1, 2, 10, 8, 4, 6, 12 },
1269							{ 10, 2, -1, 12, 14, 6, 4, 8 },
1270							{ 2, 12, 10, -1, 6, 14, 8, 4 },
1271							{ 4, 8, 14, 6, -1, 10, 12, 2 },
1272							{ 8, 4, 6, 14, 12, -1, 2, 10 },
1273							{ 10, 6, 4, 8, 12, 2, -1, 14 },
1274							{ 6, 12, 8, 4, 2, 10, 14, -1 }};
1275
1276static void kfd_set_recommended_sdma_engines(struct kfd_topology_device *to_dev,
1277					     struct kfd_iolink_properties *outbound_link,
1278					     struct kfd_iolink_properties *inbound_link)
1279{
1280	struct kfd_node *gpu = outbound_link->gpu;
1281	struct amdgpu_device *adev = gpu->adev;
1282	unsigned int num_xgmi_nodes = adev->gmc.xgmi.num_physical_nodes;
1283	unsigned int num_xgmi_sdma_engines = kfd_get_num_xgmi_sdma_engines(gpu);
1284	unsigned int num_sdma_engines = kfd_get_num_sdma_engines(gpu);
1285	uint32_t sdma_eng_id_mask = (1 << num_sdma_engines) - 1;
1286	uint32_t xgmi_sdma_eng_id_mask =
1287			((1 << num_xgmi_sdma_engines) - 1) << num_sdma_engines;
1288
1289	bool support_rec_eng = !amdgpu_sriov_vf(adev) && to_dev->gpu &&
1290		adev->aid_mask && num_xgmi_nodes && gpu->kfd->num_nodes == 1 &&
1291		num_xgmi_sdma_engines >= 6 && (!(adev->flags & AMD_IS_APU) &&
1292		num_xgmi_nodes == 8);
1293
1294	if (support_rec_eng) {
1295		int src_socket_id = adev->gmc.xgmi.physical_node_id;
1296		int dst_socket_id = to_dev->gpu->adev->gmc.xgmi.physical_node_id;
1297		unsigned int reshift = num_xgmi_sdma_engines == 6 ? 1 : 0;
1298
1299		outbound_link->rec_sdma_eng_id_mask =
1300			1 << (rec_sdma_eng_map[src_socket_id][dst_socket_id] >> reshift);
1301		inbound_link->rec_sdma_eng_id_mask =
1302			1 << (rec_sdma_eng_map[dst_socket_id][src_socket_id] >> reshift);
1303
1304		/* If recommended engine is out of range, need to reset the mask */
1305		if (outbound_link->rec_sdma_eng_id_mask & sdma_eng_id_mask)
1306			outbound_link->rec_sdma_eng_id_mask = xgmi_sdma_eng_id_mask;
1307		if (inbound_link->rec_sdma_eng_id_mask & sdma_eng_id_mask)
1308			inbound_link->rec_sdma_eng_id_mask = xgmi_sdma_eng_id_mask;
1309
1310	} else {
1311		uint32_t engine_mask = (outbound_link->iolink_type == CRAT_IOLINK_TYPE_XGMI &&
1312				num_xgmi_sdma_engines && to_dev->gpu) ? xgmi_sdma_eng_id_mask :
1313				sdma_eng_id_mask;
1314
1315		outbound_link->rec_sdma_eng_id_mask = engine_mask;
1316		inbound_link->rec_sdma_eng_id_mask = engine_mask;
1317	}
1318}
1319
1320static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
1321{
1322	struct kfd_iolink_properties *link, *inbound_link;
1323	struct kfd_topology_device *peer_dev;
1324
1325	if (!dev || !dev->gpu)
1326		return;
1327
1328	/* GPU only creates direct links so apply flags setting to all */
1329	list_for_each_entry(link, &dev->io_link_props, list) {
1330		link->flags = CRAT_IOLINK_FLAGS_ENABLED;
1331		kfd_set_iolink_no_atomics(dev, NULL, link);
1332		peer_dev = kfd_topology_device_by_proximity_domain(
1333				link->node_to);
1334
1335		if (!peer_dev)
1336			continue;
1337
1338		/* Include the CPU peer in GPU hive if connected over xGMI. */
1339		if (!peer_dev->gpu &&
1340		    link->iolink_type == CRAT_IOLINK_TYPE_XGMI) {
1341			/*
1342			 * If the GPU is not part of a GPU hive, use its pci
1343			 * device location as the hive ID to bind with the CPU.
1344			 */
1345			if (!dev->node_props.hive_id)
1346				dev->node_props.hive_id = pci_dev_id(dev->gpu->adev->pdev);
1347			peer_dev->node_props.hive_id = dev->node_props.hive_id;
1348		}
1349
1350		list_for_each_entry(inbound_link, &peer_dev->io_link_props,
1351									list) {
1352			if (inbound_link->node_to != link->node_from)
1353				continue;
1354
1355			inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED;
1356			kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link);
1357			kfd_set_iolink_non_coherent(peer_dev, link, inbound_link);
1358			kfd_set_recommended_sdma_engines(peer_dev, link, inbound_link);
1359		}
1360	}
1361
1362	/* Create indirect links so apply flags setting to all */
1363	list_for_each_entry(link, &dev->p2p_link_props, list) {
1364		link->flags = CRAT_IOLINK_FLAGS_ENABLED;
1365		kfd_set_iolink_no_atomics(dev, NULL, link);
1366		peer_dev = kfd_topology_device_by_proximity_domain(
1367				link->node_to);
1368
1369		if (!peer_dev)
1370			continue;
1371
1372		list_for_each_entry(inbound_link, &peer_dev->p2p_link_props,
1373									list) {
1374			if (inbound_link->node_to != link->node_from)
1375				continue;
1376
1377			inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED;
1378			kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link);
1379			kfd_set_iolink_non_coherent(peer_dev, link, inbound_link);
1380		}
1381	}
1382}
1383
1384static int kfd_build_p2p_node_entry(struct kfd_topology_device *dev,
1385				struct kfd_iolink_properties *p2plink)
1386{
1387	int ret;
1388
1389	p2plink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
1390	if (!p2plink->kobj)
1391		return -ENOMEM;
1392
1393	ret = kobject_init_and_add(p2plink->kobj, &iolink_type,
1394			dev->kobj_p2plink, "%d", dev->node_props.p2p_links_count - 1);
1395	if (ret < 0) {
1396		kobject_put(p2plink->kobj);
1397		return ret;
1398	}
1399
1400	p2plink->attr.name = "properties";
1401	p2plink->attr.mode = KFD_SYSFS_FILE_MODE;
1402	sysfs_attr_init(&p2plink->attr);
1403	ret = sysfs_create_file(p2plink->kobj, &p2plink->attr);
1404	if (ret < 0)
1405		return ret;
1406
1407	return 0;
1408}
1409
1410static int kfd_create_indirect_link_prop(struct kfd_topology_device *kdev, int gpu_node)
1411{
1412	struct kfd_iolink_properties *gpu_link, *tmp_link, *cpu_link;
1413	struct kfd_iolink_properties *props = NULL, *props2 = NULL;
1414	struct kfd_topology_device *cpu_dev;
1415	int ret = 0;
1416	int i, num_cpu;
1417
1418	num_cpu = 0;
1419	list_for_each_entry(cpu_dev, &topology_device_list, list) {
1420		if (cpu_dev->gpu)
1421			break;
1422		num_cpu++;
1423	}
1424
1425	if (list_empty(&kdev->io_link_props))
1426		return -ENODATA;
1427
1428	gpu_link = list_first_entry(&kdev->io_link_props,
1429				    struct kfd_iolink_properties, list);
1430
1431	for (i = 0; i < num_cpu; i++) {
1432		/* CPU <--> GPU */
1433		if (gpu_link->node_to == i)
1434			continue;
1435
1436		/* find CPU <-->  CPU links */
1437		cpu_link = NULL;
1438		cpu_dev = kfd_topology_device_by_proximity_domain(i);
1439		if (cpu_dev) {
1440			list_for_each_entry(tmp_link,
1441					&cpu_dev->io_link_props, list) {
1442				if (tmp_link->node_to == gpu_link->node_to) {
1443					cpu_link = tmp_link;
1444					break;
1445				}
1446			}
1447		}
1448
1449		if (!cpu_link)
1450			return -ENOMEM;
1451
1452		/* CPU <--> CPU <--> GPU, GPU node*/
1453		props = kfd_alloc_struct(props);
1454		if (!props)
1455			return -ENOMEM;
1456
1457		memcpy(props, gpu_link, sizeof(struct kfd_iolink_properties));
1458		props->weight = gpu_link->weight + cpu_link->weight;
1459		props->min_latency = gpu_link->min_latency + cpu_link->min_latency;
1460		props->max_latency = gpu_link->max_latency + cpu_link->max_latency;
1461		props->min_bandwidth = min(gpu_link->min_bandwidth, cpu_link->min_bandwidth);
1462		props->max_bandwidth = min(gpu_link->max_bandwidth, cpu_link->max_bandwidth);
1463
1464		props->node_from = gpu_node;
1465		props->node_to = i;
1466		kdev->node_props.p2p_links_count++;
1467		list_add_tail(&props->list, &kdev->p2p_link_props);
1468		ret = kfd_build_p2p_node_entry(kdev, props);
1469		if (ret < 0)
1470			return ret;
1471
1472		/* for small Bar, no CPU --> GPU in-direct links */
1473		if (kfd_dev_is_large_bar(kdev->gpu)) {
1474			/* CPU <--> CPU <--> GPU, CPU node*/
1475			props2 = kfd_alloc_struct(props2);
1476			if (!props2)
1477				return -ENOMEM;
1478
1479			memcpy(props2, props, sizeof(struct kfd_iolink_properties));
1480			props2->node_from = i;
1481			props2->node_to = gpu_node;
1482			props2->kobj = NULL;
1483			cpu_dev->node_props.p2p_links_count++;
1484			list_add_tail(&props2->list, &cpu_dev->p2p_link_props);
1485			ret = kfd_build_p2p_node_entry(cpu_dev, props2);
1486			if (ret < 0)
1487				return ret;
1488		}
1489	}
1490	return ret;
1491}
1492
1493#if defined(CONFIG_HSA_AMD_P2P)
1494static int kfd_add_peer_prop(struct kfd_topology_device *kdev,
1495		struct kfd_topology_device *peer, int from, int to)
1496{
1497	struct kfd_iolink_properties *props = NULL;
1498	struct kfd_iolink_properties *iolink1, *iolink2, *iolink3;
1499	struct kfd_topology_device *cpu_dev;
1500	int ret = 0;
1501
1502	if (!amdgpu_device_is_peer_accessible(
1503				kdev->gpu->adev,
1504				peer->gpu->adev))
1505		return ret;
1506
1507	if (list_empty(&kdev->io_link_props))
1508		return -ENODATA;
1509
1510	iolink1 = list_first_entry(&kdev->io_link_props,
1511				   struct kfd_iolink_properties, list);
1512
1513	if (list_empty(&peer->io_link_props))
1514		return -ENODATA;
1515
1516	iolink2 = list_first_entry(&peer->io_link_props,
1517				   struct kfd_iolink_properties, list);
1518
1519	props = kfd_alloc_struct(props);
1520	if (!props)
1521		return -ENOMEM;
1522
1523	memcpy(props, iolink1, sizeof(struct kfd_iolink_properties));
1524
1525	props->weight = iolink1->weight + iolink2->weight;
1526	props->min_latency = iolink1->min_latency + iolink2->min_latency;
1527	props->max_latency = iolink1->max_latency + iolink2->max_latency;
1528	props->min_bandwidth = min(iolink1->min_bandwidth, iolink2->min_bandwidth);
1529	props->max_bandwidth = min(iolink2->max_bandwidth, iolink2->max_bandwidth);
1530
1531	if (iolink1->node_to != iolink2->node_to) {
1532		/* CPU->CPU  link*/
1533		cpu_dev = kfd_topology_device_by_proximity_domain(iolink1->node_to);
1534		if (cpu_dev) {
1535			list_for_each_entry(iolink3, &cpu_dev->io_link_props, list) {
1536				if (iolink3->node_to != iolink2->node_to)
1537					continue;
1538
1539				props->weight += iolink3->weight;
1540				props->min_latency += iolink3->min_latency;
1541				props->max_latency += iolink3->max_latency;
1542				props->min_bandwidth = min(props->min_bandwidth,
1543							   iolink3->min_bandwidth);
1544				props->max_bandwidth = min(props->max_bandwidth,
1545							   iolink3->max_bandwidth);
1546				break;
1547			}
1548		} else {
1549			WARN(1, "CPU node not found");
1550		}
1551	}
1552
1553	props->node_from = from;
1554	props->node_to = to;
1555	peer->node_props.p2p_links_count++;
1556	list_add_tail(&props->list, &peer->p2p_link_props);
1557	ret = kfd_build_p2p_node_entry(peer, props);
1558
1559	return ret;
1560}
1561#endif
1562
1563static int kfd_dev_create_p2p_links(void)
1564{
1565	struct kfd_topology_device *dev;
1566	struct kfd_topology_device *new_dev;
1567#if defined(CONFIG_HSA_AMD_P2P)
1568	uint32_t i;
1569#endif
1570	uint32_t k;
1571	int ret = 0;
1572
1573	k = 0;
1574	list_for_each_entry(dev, &topology_device_list, list)
1575		k++;
1576	if (k < 2)
1577		return 0;
1578
1579	new_dev = list_last_entry(&topology_device_list, struct kfd_topology_device, list);
1580	if (WARN_ON(!new_dev->gpu))
1581		return 0;
1582
1583	k--;
1584
1585	/* create in-direct links */
1586	ret = kfd_create_indirect_link_prop(new_dev, k);
1587	if (ret < 0)
1588		goto out;
1589
1590	/* create p2p links */
1591#if defined(CONFIG_HSA_AMD_P2P)
1592	i = 0;
1593	list_for_each_entry(dev, &topology_device_list, list) {
1594		if (dev == new_dev)
1595			break;
1596		if (!dev->gpu || !dev->gpu->adev ||
1597		    (dev->gpu->kfd->hive_id &&
1598		     dev->gpu->kfd->hive_id == new_dev->gpu->kfd->hive_id &&
1599		     amdgpu_xgmi_get_is_sharing_enabled(dev->gpu->adev, new_dev->gpu->adev)))
1600			goto next;
1601
1602		/* check if node(s) is/are peer accessible in one direction or bi-direction */
1603		ret = kfd_add_peer_prop(new_dev, dev, i, k);
1604		if (ret < 0)
1605			goto out;
1606
1607		ret = kfd_add_peer_prop(dev, new_dev, k, i);
1608		if (ret < 0)
1609			goto out;
1610next:
1611		i++;
1612	}
1613#endif
1614
1615out:
1616	return ret;
1617}
1618
1619/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
1620static int fill_in_l1_pcache(struct kfd_cache_properties **props_ext,
1621				struct kfd_gpu_cache_info *pcache_info,
1622				int cu_bitmask,
1623				int cache_type, unsigned int cu_processor_id,
1624				int cu_block)
1625{
1626	unsigned int cu_sibling_map_mask;
1627	int first_active_cu;
1628	struct kfd_cache_properties *pcache = NULL;
1629
1630	cu_sibling_map_mask = cu_bitmask;
1631	cu_sibling_map_mask >>= cu_block;
1632	cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1);
1633	first_active_cu = ffs(cu_sibling_map_mask);
1634
1635	/* CU could be inactive. In case of shared cache find the first active
1636	 * CU. and incase of non-shared cache check if the CU is inactive. If
1637	 * inactive active skip it
1638	 */
1639	if (first_active_cu) {
1640		pcache = kfd_alloc_struct(pcache);
1641		if (!pcache)
1642			return -ENOMEM;
1643
1644		memset(pcache, 0, sizeof(struct kfd_cache_properties));
1645		pcache->processor_id_low = cu_processor_id + (first_active_cu - 1);
1646		pcache->cache_level = pcache_info[cache_type].cache_level;
1647		pcache->cache_size = pcache_info[cache_type].cache_size;
1648		pcache->cacheline_size = pcache_info[cache_type].cache_line_size;
1649
1650		if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE)
1651			pcache->cache_type |= HSA_CACHE_TYPE_DATA;
1652		if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE)
1653			pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
1654		if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE)
1655			pcache->cache_type |= HSA_CACHE_TYPE_CPU;
1656		if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
1657			pcache->cache_type |= HSA_CACHE_TYPE_HSACU;
1658
1659		/* Sibling map is w.r.t processor_id_low, so shift out
1660		 * inactive CU
1661		 */
1662		cu_sibling_map_mask =
1663			cu_sibling_map_mask >> (first_active_cu - 1);
1664
1665		pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF);
1666		pcache->sibling_map[1] =
1667				(uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
1668		pcache->sibling_map[2] =
1669				(uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
1670		pcache->sibling_map[3] =
1671				(uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
1672
1673		pcache->sibling_map_size = 4;
1674		*props_ext = pcache;
1675
1676		return 0;
1677	}
1678	return 1;
1679}
1680
1681/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
1682static int fill_in_l2_l3_pcache(struct kfd_cache_properties **props_ext,
1683				struct kfd_gpu_cache_info *pcache_info,
1684				struct amdgpu_cu_info *cu_info,
1685				struct amdgpu_gfx_config *gfx_info,
1686				int cache_type, unsigned int cu_processor_id,
1687				struct kfd_node *knode)
1688{
1689	unsigned int cu_sibling_map_mask = 0;
1690	int first_active_cu;
1691	int i, j, k, xcc, start, end;
1692	int num_xcc = NUM_XCC(knode->xcc_mask);
1693	struct kfd_cache_properties *pcache = NULL;
1694	enum amdgpu_memory_partition mode;
1695	struct amdgpu_device *adev = knode->adev;
1696	bool found = false;
1697
1698	start = ffs(knode->xcc_mask) - 1;
1699	end = start + num_xcc;
1700
1701	/* To find the bitmap in the first active cu in the first
1702	 * xcc, it is based on the assumption that evrey xcc must
1703	 * have at least one active cu.
1704	 */
1705	for (i = 0; i < gfx_info->max_shader_engines && !found; i++) {
1706		for (j = 0; j < gfx_info->max_sh_per_se && !found; j++) {
1707			if (cu_info->bitmap[start][i % 4][j % 4]) {
1708				cu_sibling_map_mask =
1709					cu_info->bitmap[start][i % 4][j % 4];
1710				found = true;
1711			}
1712		}
1713	}
1714
1715	cu_sibling_map_mask &=
1716		((1 << pcache_info[cache_type].num_cu_shared) - 1);
1717	first_active_cu = ffs(cu_sibling_map_mask);
1718
1719	/* CU could be inactive. In case of shared cache find the first active
1720	 * CU. and incase of non-shared cache check if the CU is inactive. If
1721	 * inactive active skip it
1722	 */
1723	if (first_active_cu) {
1724		pcache = kfd_alloc_struct(pcache);
1725		if (!pcache)
1726			return -ENOMEM;
1727
1728		memset(pcache, 0, sizeof(struct kfd_cache_properties));
1729		pcache->processor_id_low = cu_processor_id
1730					+ (first_active_cu - 1);
1731		pcache->cache_level = pcache_info[cache_type].cache_level;
1732		pcache->cacheline_size = pcache_info[cache_type].cache_line_size;
1733
1734		if (KFD_GC_VERSION(knode) == IP_VERSION(9, 4, 3) ||
1735		    KFD_GC_VERSION(knode) == IP_VERSION(9, 4, 4) ||
1736		    KFD_GC_VERSION(knode) == IP_VERSION(9, 5, 0))
1737			mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
1738		else
1739			mode = UNKNOWN_MEMORY_PARTITION_MODE;
1740
1741		pcache->cache_size = pcache_info[cache_type].cache_size;
1742		/* Partition mode only affects L3 cache size */
1743		if (mode && pcache->cache_level == 3)
1744			pcache->cache_size /= mode;
1745
1746		if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE)
1747			pcache->cache_type |= HSA_CACHE_TYPE_DATA;
1748		if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE)
1749			pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
1750		if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE)
1751			pcache->cache_type |= HSA_CACHE_TYPE_CPU;
1752		if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
1753			pcache->cache_type |= HSA_CACHE_TYPE_HSACU;
1754
1755		/* Sibling map is w.r.t processor_id_low, so shift out
1756		 * inactive CU
1757		 */
1758		cu_sibling_map_mask = cu_sibling_map_mask >> (first_active_cu - 1);
1759		k = 0;
1760
1761		for (xcc = start; xcc < end; xcc++) {
1762			for (i = 0; i < gfx_info->max_shader_engines; i++) {
1763				for (j = 0; j < gfx_info->max_sh_per_se; j++) {
1764					pcache->sibling_map[k] = (uint8_t)(cu_sibling_map_mask & 0xFF);
1765					pcache->sibling_map[k+1] = (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
1766					pcache->sibling_map[k+2] = (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
1767					pcache->sibling_map[k+3] = (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
1768					k += 4;
1769
1770					cu_sibling_map_mask = cu_info->bitmap[xcc][i % 4][j + i / 4];
1771					cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1);
1772				}
1773			}
1774		}
1775		pcache->sibling_map_size = k;
1776		*props_ext = pcache;
1777		return 0;
1778	}
1779	return 1;
1780}
1781
1782#define KFD_MAX_CACHE_TYPES 6
1783
1784/* kfd_fill_cache_non_crat_info - Fill GPU cache info using kfd_gpu_cache_info
1785 * tables
1786 */
1787static void kfd_fill_cache_non_crat_info(struct kfd_topology_device *dev, struct kfd_node *kdev)
1788{
1789	struct kfd_gpu_cache_info *pcache_info = NULL;
1790	int i, j, k, xcc, start, end;
1791	int ct = 0;
1792	unsigned int cu_processor_id;
1793	int ret;
1794	unsigned int num_cu_shared;
1795	struct amdgpu_cu_info *cu_info = &kdev->adev->gfx.cu_info;
1796	struct amdgpu_gfx_config *gfx_info = &kdev->adev->gfx.config;
1797	int gpu_processor_id;
1798	struct kfd_cache_properties *props_ext = NULL;
1799	int num_of_entries = 0;
1800	int num_of_cache_types = 0;
1801	struct kfd_gpu_cache_info cache_info[KFD_MAX_CACHE_TYPES];
1802
1803
1804	gpu_processor_id = dev->node_props.simd_id_base;
1805
1806	memset(cache_info, 0, sizeof(cache_info));
1807	pcache_info = cache_info;
1808	num_of_cache_types = kfd_get_gpu_cache_info(kdev, &pcache_info);
1809	if (!num_of_cache_types) {
1810		pr_warn("no cache info found\n");
1811		return;
1812	}
1813
1814	/* For each type of cache listed in the kfd_gpu_cache_info table,
1815	 * go through all available Compute Units.
1816	 * The [i,j,k] loop will
1817	 *		if kfd_gpu_cache_info.num_cu_shared = 1
1818	 *			will parse through all available CU
1819	 *		If (kfd_gpu_cache_info.num_cu_shared != 1)
1820	 *			then it will consider only one CU from
1821	 *			the shared unit
1822	 */
1823	start = ffs(kdev->xcc_mask) - 1;
1824	end = start + NUM_XCC(kdev->xcc_mask);
1825
1826	for (ct = 0; ct < num_of_cache_types; ct++) {
1827		cu_processor_id = gpu_processor_id;
1828		if (pcache_info[ct].cache_level == 1) {
1829			for (xcc = start; xcc < end; xcc++) {
1830				for (i = 0; i < gfx_info->max_shader_engines; i++) {
1831					for (j = 0; j < gfx_info->max_sh_per_se; j++) {
1832						for (k = 0; k < gfx_info->max_cu_per_sh; k += pcache_info[ct].num_cu_shared) {
1833
1834							ret = fill_in_l1_pcache(&props_ext, pcache_info,
1835										cu_info->bitmap[xcc][i % 4][j + i / 4], ct,
1836										cu_processor_id, k);
1837
1838							if (ret < 0)
1839								break;
1840
1841							if (!ret) {
1842								num_of_entries++;
1843								list_add_tail(&props_ext->list, &dev->cache_props);
1844							}
1845
1846							/* Move to next CU block */
1847							num_cu_shared = ((k + pcache_info[ct].num_cu_shared) <=
1848								gfx_info->max_cu_per_sh) ?
1849								pcache_info[ct].num_cu_shared :
1850								(gfx_info->max_cu_per_sh - k);
1851							cu_processor_id += num_cu_shared;
1852						}
1853					}
1854				}
1855			}
1856		} else {
1857			ret = fill_in_l2_l3_pcache(&props_ext, pcache_info,
1858						   cu_info, gfx_info, ct, cu_processor_id, kdev);
1859
1860			if (ret < 0)
1861				break;
1862
1863			if (!ret) {
1864				num_of_entries++;
1865				list_add_tail(&props_ext->list, &dev->cache_props);
1866			}
1867		}
1868	}
1869	dev->node_props.caches_count += num_of_entries;
1870	pr_debug("Added [%d] GPU cache entries\n", num_of_entries);
1871}
1872
1873static int kfd_topology_add_device_locked(struct kfd_node *gpu,
1874					  struct kfd_topology_device **dev)
1875{
1876	int proximity_domain = ++topology_crat_proximity_domain;
1877	struct list_head temp_topology_device_list;
1878	void *crat_image = NULL;
1879	size_t image_size = 0;
1880	int res;
1881
1882	res = kfd_create_crat_image_virtual(&crat_image, &image_size,
1883					    COMPUTE_UNIT_GPU, gpu,
1884					    proximity_domain);
1885	if (res) {
1886		dev_err(gpu->adev->dev, "Error creating VCRAT\n");
1887		topology_crat_proximity_domain--;
1888		goto err;
1889	}
1890
1891	INIT_LIST_HEAD(&temp_topology_device_list);
1892
1893	res = kfd_parse_crat_table(crat_image,
1894				   &temp_topology_device_list,
1895				   proximity_domain);
1896	if (res) {
1897		dev_err(gpu->adev->dev, "Error parsing VCRAT\n");
1898		topology_crat_proximity_domain--;
1899		goto err;
1900	}
1901
1902	kfd_topology_update_device_list(&temp_topology_device_list,
1903					&topology_device_list);
1904
1905	*dev = kfd_assign_gpu(gpu);
1906	if (WARN_ON(!*dev)) {
1907		res = -ENODEV;
1908		goto err;
1909	}
1910
1911	/* Fill the cache affinity information here for the GPUs
1912	 * using VCRAT
1913	 */
1914	kfd_fill_cache_non_crat_info(*dev, gpu);
1915
1916	/* Update the SYSFS tree, since we added another topology
1917	 * device
1918	 */
1919	res = kfd_topology_update_sysfs();
1920	if (!res)
1921		sys_props.generation_count++;
1922	else
1923		dev_err(gpu->adev->dev, "Failed to update GPU to sysfs topology. res=%d\n",
1924			res);
1925
1926err:
1927	kfd_destroy_crat_image(crat_image);
1928	return res;
1929}
1930
1931static void kfd_topology_set_dbg_firmware_support(struct kfd_topology_device *dev)
1932{
1933	bool firmware_supported = true;
1934
1935	if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(11, 0, 0) &&
1936			KFD_GC_VERSION(dev->gpu) < IP_VERSION(12, 0, 0)) {
1937		uint32_t mes_api_rev = (dev->gpu->adev->mes.sched_version &
1938						AMDGPU_MES_API_VERSION_MASK) >>
1939						AMDGPU_MES_API_VERSION_SHIFT;
1940		uint32_t mes_rev = dev->gpu->adev->mes.sched_version &
1941						AMDGPU_MES_VERSION_MASK;
1942
1943		firmware_supported = (mes_api_rev >= 14) && (mes_rev >= 64);
1944		goto out;
1945	}
1946
1947	/*
1948	 * Note: Any unlisted devices here are assumed to support exception handling.
1949	 * Add additional checks here as needed.
1950	 */
1951	switch (KFD_GC_VERSION(dev->gpu)) {
1952	case IP_VERSION(9, 0, 1):
1953		firmware_supported = dev->gpu->kfd->mec_fw_version >= 459 + 32768;
1954		break;
1955	case IP_VERSION(9, 1, 0):
1956	case IP_VERSION(9, 2, 1):
1957	case IP_VERSION(9, 2, 2):
1958	case IP_VERSION(9, 3, 0):
1959	case IP_VERSION(9, 4, 0):
1960		firmware_supported = dev->gpu->kfd->mec_fw_version >= 459;
1961		break;
1962	case IP_VERSION(9, 4, 1):
1963		firmware_supported = dev->gpu->kfd->mec_fw_version >= 60;
1964		break;
1965	case IP_VERSION(9, 4, 2):
1966		firmware_supported = dev->gpu->kfd->mec_fw_version >= 51;
1967		break;
1968	case IP_VERSION(10, 1, 10):
1969	case IP_VERSION(10, 1, 2):
1970	case IP_VERSION(10, 1, 1):
1971		firmware_supported = dev->gpu->kfd->mec_fw_version >= 144;
1972		break;
1973	case IP_VERSION(10, 3, 0):
1974	case IP_VERSION(10, 3, 2):
1975	case IP_VERSION(10, 3, 1):
1976	case IP_VERSION(10, 3, 4):
1977	case IP_VERSION(10, 3, 5):
1978		firmware_supported = dev->gpu->kfd->mec_fw_version >= 89;
1979		break;
1980	case IP_VERSION(10, 1, 3):
1981	case IP_VERSION(10, 3, 3):
1982		firmware_supported = false;
1983		break;
1984	default:
1985		break;
1986	}
1987
1988out:
1989	if (firmware_supported)
1990		dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_FIRMWARE_SUPPORTED;
1991}
1992
1993static void kfd_topology_set_capabilities(struct kfd_topology_device *dev)
1994{
1995	dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 <<
1996				HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
1997				HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
1998
1999	dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_SUPPORT |
2000			HSA_CAP_TRAP_DEBUG_WAVE_LAUNCH_TRAP_OVERRIDE_SUPPORTED |
2001			HSA_CAP_TRAP_DEBUG_WAVE_LAUNCH_MODE_SUPPORTED;
2002
2003	if (kfd_dbg_has_ttmps_always_setup(dev->gpu))
2004		dev->node_props.debug_prop |= HSA_DBG_DISPATCH_INFO_ALWAYS_VALID;
2005
2006	if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0)) {
2007		if (KFD_GC_VERSION(dev->gpu) == IP_VERSION(9, 4, 3) ||
2008		    KFD_GC_VERSION(dev->gpu) == IP_VERSION(9, 4, 4))
2009			dev->node_props.debug_prop |=
2010				HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX9_4_3 |
2011				HSA_DBG_WATCH_ADDR_MASK_HI_BIT_GFX9_4_3;
2012		else
2013			dev->node_props.debug_prop |=
2014				HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX9 |
2015				HSA_DBG_WATCH_ADDR_MASK_HI_BIT;
2016
2017		if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(9, 4, 2))
2018			dev->node_props.capability |=
2019				HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED;
2020
2021		if (!amdgpu_sriov_vf(dev->gpu->adev))
2022			dev->node_props.capability |= HSA_CAP_PER_QUEUE_RESET_SUPPORTED;
2023
2024	} else {
2025		dev->node_props.debug_prop |= HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX10 |
2026					HSA_DBG_WATCH_ADDR_MASK_HI_BIT;
2027
2028		if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(12, 0, 0))
2029			dev->node_props.capability |=
2030				HSA_CAP_TRAP_DEBUG_PRECISE_ALU_OPERATIONS_SUPPORTED;
2031	}
2032
2033	kfd_topology_set_dbg_firmware_support(dev);
2034}
2035
2036int kfd_topology_add_device(struct kfd_node *gpu)
2037{
2038	uint32_t gpu_id;
2039	struct kfd_topology_device *dev;
2040	int res = 0;
2041	int i;
2042	const char *asic_name = amdgpu_asic_name[gpu->adev->asic_type];
2043	struct amdgpu_gfx_config *gfx_info = &gpu->adev->gfx.config;
2044	struct amdgpu_cu_info *cu_info = &gpu->adev->gfx.cu_info;
2045
2046	if (gpu->xcp && !gpu->xcp->ddev) {
2047		dev_warn(gpu->adev->dev,
2048			 "Won't add GPU to topology since it has no drm node assigned.");
2049		return 0;
2050	} else {
2051		dev_dbg(gpu->adev->dev, "Adding new GPU to topology\n");
2052	}
2053
2054	/* Check to see if this gpu device exists in the topology_device_list.
2055	 * If so, assign the gpu to that device,
2056	 * else create a Virtual CRAT for this gpu device and then parse that
2057	 * CRAT to create a new topology device. Once created assign the gpu to
2058	 * that topology device
2059	 */
2060	down_write(&topology_lock);
2061	dev = kfd_assign_gpu(gpu);
2062	if (!dev)
2063		res = kfd_topology_add_device_locked(gpu, &dev);
2064	up_write(&topology_lock);
2065	if (res)
2066		return res;
2067
2068	gpu_id = kfd_generate_gpu_id(gpu);
2069	dev->gpu_id = gpu_id;
2070	gpu->id = gpu_id;
2071
2072	kfd_dev_create_p2p_links();
2073
2074	/* TODO: Move the following lines to function
2075	 *	kfd_add_non_crat_information
2076	 */
2077
2078	/* Fill-in additional information that is not available in CRAT but
2079	 * needed for the topology
2080	 */
2081	for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1; i++) {
2082		dev->node_props.name[i] = __tolower(asic_name[i]);
2083		if (asic_name[i] == '\0')
2084			break;
2085	}
2086	dev->node_props.name[i] = '\0';
2087
2088	dev->node_props.simd_arrays_per_engine =
2089		gfx_info->max_sh_per_se;
2090
2091	dev->node_props.gfx_target_version =
2092				gpu->kfd->device_info.gfx_target_version;
2093	dev->node_props.vendor_id = gpu->adev->pdev->vendor;
2094	dev->node_props.device_id = gpu->adev->pdev->device;
2095	dev->node_props.capability |=
2096		((dev->gpu->adev->rev_id << HSA_CAP_ASIC_REVISION_SHIFT) &
2097			HSA_CAP_ASIC_REVISION_MASK);
2098
2099	dev->node_props.location_id = pci_dev_id(gpu->adev->pdev);
2100	if (gpu->kfd->num_nodes > 1)
2101		dev->node_props.location_id |= dev->gpu->node_id;
2102
2103	dev->node_props.domain = pci_domain_nr(gpu->adev->pdev->bus);
2104	dev->node_props.max_engine_clk_fcompute =
2105		amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->adev);
2106	dev->node_props.max_engine_clk_ccompute =
2107		cpufreq_quick_get_max(0) / 1000;
2108
2109	if (gpu->xcp)
2110		dev->node_props.drm_render_minor = gpu->xcp->ddev->render->index;
2111	else
2112		dev->node_props.drm_render_minor =
2113				gpu->kfd->shared_resources.drm_render_minor;
2114
2115	dev->node_props.hive_id = gpu->kfd->hive_id;
2116	dev->node_props.num_sdma_engines = kfd_get_num_sdma_engines(gpu);
2117	dev->node_props.num_sdma_xgmi_engines =
2118					kfd_get_num_xgmi_sdma_engines(gpu);
2119	dev->node_props.num_sdma_queues_per_engine =
2120				gpu->kfd->device_info.num_sdma_queues_per_engine -
2121				gpu->kfd->device_info.num_reserved_sdma_queues_per_engine;
2122	dev->node_props.num_gws = (dev->gpu->gws &&
2123		dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ?
2124		dev->gpu->adev->gds.gws_size : 0;
2125	dev->node_props.num_cp_queues = get_cp_queues_num(dev->gpu->dqm);
2126
2127	kfd_fill_mem_clk_max_info(dev);
2128	kfd_fill_iolink_non_crat_info(dev);
2129
2130	switch (dev->gpu->adev->asic_type) {
2131	case CHIP_KAVERI:
2132	case CHIP_HAWAII:
2133	case CHIP_TONGA:
2134		dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 <<
2135			HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
2136			HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
2137		break;
2138	case CHIP_CARRIZO:
2139	case CHIP_FIJI:
2140	case CHIP_POLARIS10:
2141	case CHIP_POLARIS11:
2142	case CHIP_POLARIS12:
2143	case CHIP_VEGAM:
2144		pr_debug("Adding doorbell packet type capability\n");
2145		dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 <<
2146			HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
2147			HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
2148		break;
2149	default:
2150		if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(9, 0, 1))
2151			WARN(1, "Unexpected ASIC family %u",
2152			     dev->gpu->adev->asic_type);
2153		else
2154			kfd_topology_set_capabilities(dev);
2155	}
2156
2157	/*
2158	 * Overwrite ATS capability according to needs_iommu_device to fix
2159	 * potential missing corresponding bit in CRAT of BIOS.
2160	 */
2161	dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT;
2162
2163	/* Fix errors in CZ CRAT.
2164	 * simd_count: Carrizo CRAT reports wrong simd_count, probably
2165	 *		because it doesn't consider masked out CUs
2166	 * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd
2167	 */
2168	if (dev->gpu->adev->asic_type == CHIP_CARRIZO) {
2169		dev->node_props.simd_count =
2170			cu_info->simd_per_cu * cu_info->number;
2171		dev->node_props.max_waves_per_simd = 10;
2172	}
2173
2174	/* kfd only concerns sram ecc on GFX and HBM ecc on UMC */
2175	dev->node_props.capability |=
2176		((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ?
2177		HSA_CAP_SRAM_EDCSUPPORTED : 0;
2178	dev->node_props.capability |=
2179		((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?
2180		HSA_CAP_MEM_EDCSUPPORTED : 0;
2181
2182	if (KFD_GC_VERSION(dev->gpu) != IP_VERSION(9, 0, 1))
2183		dev->node_props.capability |= (dev->gpu->adev->ras_enabled != 0) ?
2184			HSA_CAP_RASEVENTNOTIFY : 0;
2185
2186	if (KFD_IS_SVM_API_SUPPORTED(dev->gpu->adev))
2187		dev->node_props.capability |= HSA_CAP_SVMAPI_SUPPORTED;
2188
2189	if (dev->gpu->adev->gmc.is_app_apu ||
2190		dev->gpu->adev->gmc.xgmi.connected_to_cpu)
2191		dev->node_props.capability |= HSA_CAP_FLAGS_COHERENTHOSTACCESS;
2192
2193	kfd_queue_ctx_save_restore_size(dev);
2194
2195	kfd_debug_print_topology();
2196
2197	kfd_notify_gpu_change(gpu_id, 1);
2198
2199	return 0;
2200}
2201
2202/**
2203 * kfd_topology_update_io_links() - Update IO links after device removal.
2204 * @proximity_domain: Proximity domain value of the dev being removed.
2205 *
2206 * The topology list currently is arranged in increasing order of
2207 * proximity domain.
2208 *
2209 * Two things need to be done when a device is removed:
2210 * 1. All the IO links to this device need to be removed.
2211 * 2. All nodes after the current device node need to move
2212 *    up once this device node is removed from the topology
2213 *    list. As a result, the proximity domain values for
2214 *    all nodes after the node being deleted reduce by 1.
2215 *    This would also cause the proximity domain values for
2216 *    io links to be updated based on new proximity domain
2217 *    values.
2218 *
2219 * Context: The caller must hold write topology_lock.
2220 */
2221static void kfd_topology_update_io_links(int proximity_domain)
2222{
2223	struct kfd_topology_device *dev;
2224	struct kfd_iolink_properties *iolink, *p2plink, *tmp;
2225
2226	list_for_each_entry(dev, &topology_device_list, list) {
2227		if (dev->proximity_domain > proximity_domain)
2228			dev->proximity_domain--;
2229
2230		list_for_each_entry_safe(iolink, tmp, &dev->io_link_props, list) {
2231			/*
2232			 * If there is an io link to the dev being deleted
2233			 * then remove that IO link also.
2234			 */
2235			if (iolink->node_to == proximity_domain) {
2236				list_del(&iolink->list);
2237				dev->node_props.io_links_count--;
2238			} else {
2239				if (iolink->node_from > proximity_domain)
2240					iolink->node_from--;
2241				if (iolink->node_to > proximity_domain)
2242					iolink->node_to--;
2243			}
2244		}
2245
2246		list_for_each_entry_safe(p2plink, tmp, &dev->p2p_link_props, list) {
2247			/*
2248			 * If there is a p2p link to the dev being deleted
2249			 * then remove that p2p link also.
2250			 */
2251			if (p2plink->node_to == proximity_domain) {
2252				list_del(&p2plink->list);
2253				dev->node_props.p2p_links_count--;
2254			} else {
2255				if (p2plink->node_from > proximity_domain)
2256					p2plink->node_from--;
2257				if (p2plink->node_to > proximity_domain)
2258					p2plink->node_to--;
2259			}
2260		}
2261	}
2262}
2263
2264int kfd_topology_remove_device(struct kfd_node *gpu)
2265{
2266	struct kfd_topology_device *dev, *tmp;
2267	uint32_t gpu_id;
2268	int res = -ENODEV;
2269	int i = 0;
2270
2271	down_write(&topology_lock);
2272
2273	list_for_each_entry_safe(dev, tmp, &topology_device_list, list) {
2274		if (dev->gpu == gpu) {
2275			gpu_id = dev->gpu_id;
2276			kfd_remove_sysfs_node_entry(dev);
2277			kfd_release_topology_device(dev);
2278			sys_props.num_devices--;
2279			kfd_topology_update_io_links(i);
2280			topology_crat_proximity_domain = sys_props.num_devices-1;
2281			sys_props.generation_count++;
2282			res = 0;
2283			if (kfd_topology_update_sysfs() < 0)
2284				kfd_topology_release_sysfs();
2285			break;
2286		}
2287		i++;
2288	}
2289
2290	up_write(&topology_lock);
2291
2292	if (!res)
2293		kfd_notify_gpu_change(gpu_id, 0);
2294
2295	return res;
2296}
2297
2298/* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD
2299 *	topology. If GPU device is found @idx, then valid kfd_dev pointer is
2300 *	returned through @kdev
2301 * Return -	0: On success (@kdev will be NULL for non GPU nodes)
2302 *		-1: If end of list
2303 */
2304int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_node **kdev)
2305{
2306
2307	struct kfd_topology_device *top_dev;
2308	uint8_t device_idx = 0;
2309
2310	*kdev = NULL;
2311	down_read(&topology_lock);
2312
2313	list_for_each_entry(top_dev, &topology_device_list, list) {
2314		if (device_idx == idx) {
2315			*kdev = top_dev->gpu;
2316			up_read(&topology_lock);
2317			return 0;
2318		}
2319
2320		device_idx++;
2321	}
2322
2323	up_read(&topology_lock);
2324
2325	return -1;
2326
2327}
2328
2329static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask)
2330{
2331	int first_cpu_of_numa_node;
2332
2333	if (!cpumask || cpumask == cpu_none_mask)
2334		return -1;
2335	first_cpu_of_numa_node = cpumask_first(cpumask);
2336	if (first_cpu_of_numa_node >= nr_cpu_ids)
2337		return -1;
2338#ifdef CONFIG_X86_64
2339	return cpu_data(first_cpu_of_numa_node).topo.apicid;
2340#else
2341	return first_cpu_of_numa_node;
2342#endif
2343}
2344
2345/* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor
2346 *	of the given NUMA node (numa_node_id)
2347 * Return -1 on failure
2348 */
2349int kfd_numa_node_to_apic_id(int numa_node_id)
2350{
2351	if (numa_node_id == -1) {
2352		pr_warn("Invalid NUMA Node. Use online CPU mask\n");
2353		return kfd_cpumask_to_apic_id(cpu_online_mask);
2354	}
2355	return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id));
2356}
2357
2358#if defined(CONFIG_DEBUG_FS)
2359
2360int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data)
2361{
2362	struct kfd_topology_device *dev;
2363	unsigned int i = 0;
2364	int r = 0;
2365
2366	down_read(&topology_lock);
2367
2368	list_for_each_entry(dev, &topology_device_list, list) {
2369		if (!dev->gpu) {
2370			i++;
2371			continue;
2372		}
2373
2374		seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
2375		r = dqm_debugfs_hqds(m, dev->gpu->dqm);
2376		if (r)
2377			break;
2378	}
2379
2380	up_read(&topology_lock);
2381
2382	return r;
2383}
2384
2385int kfd_debugfs_rls_by_device(struct seq_file *m, void *data)
2386{
2387	struct kfd_topology_device *dev;
2388	unsigned int i = 0;
2389	int r = 0;
2390
2391	down_read(&topology_lock);
2392
2393	list_for_each_entry(dev, &topology_device_list, list) {
2394		if (!dev->gpu) {
2395			i++;
2396			continue;
2397		}
2398
2399		seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
2400		r = pm_debugfs_runlist(m, &dev->gpu->dqm->packet_mgr);
2401		if (r)
2402			break;
2403	}
2404
2405	up_read(&topology_lock);
2406
2407	return r;
2408}
2409
2410#endif