FILE
xe_oa.c
File preview with syntax highlighting.
linux-6.19.14/drivers/gpu/drm/xe/xe_oa.c
1// SPDX-License-Identifier: MIT 2/* 3 * Copyright © 2023-2024 Intel Corporation 4 */ 5 6#include <linux/anon_inodes.h> 7#include <linux/delay.h> 8#include <linux/nospec.h> 9#include <linux/poll.h> 10 11#include <drm/drm_drv.h> 12#include <drm/drm_managed.h> 13#include <drm/drm_syncobj.h> 14#include <uapi/drm/xe_drm.h> 15 16#include <generated/xe_wa_oob.h> 17 18#include "abi/guc_actions_slpc_abi.h" 19#include "instructions/xe_mi_commands.h" 20#include "regs/xe_engine_regs.h" 21#include "regs/xe_gt_regs.h" 22#include "regs/xe_oa_regs.h" 23#include "xe_assert.h" 24#include "xe_bb.h" 25#include "xe_bo.h" 26#include "xe_device.h" 27#include "xe_exec_queue.h" 28#include "xe_force_wake.h" 29#include "xe_gt.h" 30#include "xe_gt_mcr.h" 31#include "xe_gt_printk.h" 32#include "xe_guc_pc.h" 33#include "xe_macros.h" 34#include "xe_mmio.h" 35#include "xe_oa.h" 36#include "xe_observation.h" 37#include "xe_pm.h" 38#include "xe_sched_job.h" 39#include "xe_sriov.h" 40#include "xe_sync.h" 41#include "xe_wa.h" 42 43#define DEFAULT_POLL_FREQUENCY_HZ 200 44#define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ) 45#define XE_OA_UNIT_INVALID U32_MAX 46 47enum xe_oam_unit_type { 48 XE_OAM_UNIT_SAG, 49 XE_OAM_UNIT_SCMI_0, 50 XE_OAM_UNIT_SCMI_1, 51}; 52 53enum xe_oa_submit_deps { 54 XE_OA_SUBMIT_NO_DEPS, 55 XE_OA_SUBMIT_ADD_DEPS, 56}; 57 58enum xe_oa_user_extn_from { 59 XE_OA_USER_EXTN_FROM_OPEN, 60 XE_OA_USER_EXTN_FROM_CONFIG, 61}; 62 63struct xe_oa_reg { 64 struct xe_reg addr; 65 u32 value; 66}; 67 68struct xe_oa_config { 69 struct xe_oa *oa; 70 71 char uuid[UUID_STRING_LEN + 1]; 72 int id; 73 74 const struct xe_oa_reg *regs; 75 u32 regs_len; 76 77 struct attribute_group sysfs_metric; 78 struct attribute *attrs[2]; 79 struct kobj_attribute sysfs_metric_id; 80 81 struct kref ref; 82 struct rcu_head rcu; 83}; 84 85struct xe_oa_open_param { 86 struct xe_file *xef; 87 struct xe_oa_unit *oa_unit; 88 bool sample; 89 u32 metric_set; 90 enum xe_oa_format_name oa_format; 91 int period_exponent; 92 bool disabled; 93 int exec_queue_id; 94 int engine_instance; 95 struct xe_exec_queue *exec_q; 96 struct xe_hw_engine *hwe; 97 bool no_preempt; 98 struct drm_xe_sync __user *syncs_user; 99 int num_syncs; 100 struct xe_sync_entry *syncs; 101 size_t oa_buffer_size; 102 int wait_num_reports; 103}; 104 105struct xe_oa_config_bo { 106 struct llist_node node; 107 108 struct xe_oa_config *oa_config; 109 struct xe_bb *bb; 110}; 111 112struct xe_oa_fence { 113 /* @base: dma fence base */ 114 struct dma_fence base; 115 /* @lock: lock for the fence */ 116 spinlock_t lock; 117 /* @work: work to signal @base */ 118 struct delayed_work work; 119 /* @cb: callback to schedule @work */ 120 struct dma_fence_cb cb; 121}; 122 123#define DRM_FMT(x) DRM_XE_OA_FMT_TYPE_##x 124 125static const struct xe_oa_format oa_formats[] = { 126 [XE_OA_FORMAT_C4_B8] = { 7, 64, DRM_FMT(OAG) }, 127 [XE_OA_FORMAT_A12] = { 0, 64, DRM_FMT(OAG) }, 128 [XE_OA_FORMAT_A12_B8_C8] = { 2, 128, DRM_FMT(OAG) }, 129 [XE_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256, DRM_FMT(OAG) }, 130 [XE_OAR_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256, DRM_FMT(OAR) }, 131 [XE_OA_FORMAT_A24u40_A14u32_B8_C8] = { 5, 256, DRM_FMT(OAG) }, 132 [XE_OAC_FORMAT_A24u64_B8_C8] = { 1, 320, DRM_FMT(OAC), HDR_64_BIT }, 133 [XE_OAC_FORMAT_A22u32_R2u32_B8_C8] = { 2, 192, DRM_FMT(OAC), HDR_64_BIT }, 134 [XE_OAM_FORMAT_MPEC8u64_B8_C8] = { 1, 192, DRM_FMT(OAM_MPEC), HDR_64_BIT }, 135 [XE_OAM_FORMAT_MPEC8u32_B8_C8] = { 2, 128, DRM_FMT(OAM_MPEC), HDR_64_BIT }, 136 [XE_OA_FORMAT_PEC64u64] = { 1, 576, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, 137 [XE_OA_FORMAT_PEC64u64_B8_C8] = { 1, 640, DRM_FMT(PEC), HDR_64_BIT, 1, 1 }, 138 [XE_OA_FORMAT_PEC64u32] = { 1, 320, DRM_FMT(PEC), HDR_64_BIT }, 139 [XE_OA_FORMAT_PEC32u64_G1] = { 5, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, 140 [XE_OA_FORMAT_PEC32u32_G1] = { 5, 192, DRM_FMT(PEC), HDR_64_BIT }, 141 [XE_OA_FORMAT_PEC32u64_G2] = { 6, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, 142 [XE_OA_FORMAT_PEC32u32_G2] = { 6, 192, DRM_FMT(PEC), HDR_64_BIT }, 143 [XE_OA_FORMAT_PEC36u64_G1_32_G2_4] = { 3, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, 144 [XE_OA_FORMAT_PEC36u64_G1_4_G2_32] = { 4, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, 145}; 146 147static u32 xe_oa_circ_diff(struct xe_oa_stream *stream, u32 tail, u32 head) 148{ 149 return tail >= head ? tail - head : 150 tail + stream->oa_buffer.circ_size - head; 151} 152 153static u32 xe_oa_circ_incr(struct xe_oa_stream *stream, u32 ptr, u32 n) 154{ 155 return ptr + n >= stream->oa_buffer.circ_size ? 156 ptr + n - stream->oa_buffer.circ_size : ptr + n; 157} 158 159static void xe_oa_config_release(struct kref *ref) 160{ 161 struct xe_oa_config *oa_config = 162 container_of(ref, typeof(*oa_config), ref); 163 164 kfree(oa_config->regs); 165 166 kfree_rcu(oa_config, rcu); 167} 168 169static void xe_oa_config_put(struct xe_oa_config *oa_config) 170{ 171 if (!oa_config) 172 return; 173 174 kref_put(&oa_config->ref, xe_oa_config_release); 175} 176 177static struct xe_oa_config *xe_oa_config_get(struct xe_oa_config *oa_config) 178{ 179 return kref_get_unless_zero(&oa_config->ref) ? oa_config : NULL; 180} 181 182static struct xe_oa_config *xe_oa_get_oa_config(struct xe_oa *oa, int metrics_set) 183{ 184 struct xe_oa_config *oa_config; 185 186 rcu_read_lock(); 187 oa_config = idr_find(&oa->metrics_idr, metrics_set); 188 if (oa_config) 189 oa_config = xe_oa_config_get(oa_config); 190 rcu_read_unlock(); 191 192 return oa_config; 193} 194 195static void free_oa_config_bo(struct xe_oa_config_bo *oa_bo, struct dma_fence *last_fence) 196{ 197 xe_oa_config_put(oa_bo->oa_config); 198 xe_bb_free(oa_bo->bb, last_fence); 199 kfree(oa_bo); 200} 201 202static const struct xe_oa_regs *__oa_regs(struct xe_oa_stream *stream) 203{ 204 return &stream->oa_unit->regs; 205} 206 207static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream) 208{ 209 return xe_mmio_read32(&stream->gt->mmio, __oa_regs(stream)->oa_tail_ptr) & 210 OAG_OATAILPTR_MASK; 211} 212 213#define oa_report_header_64bit(__s) \ 214 ((__s)->oa_buffer.format->header == HDR_64_BIT) 215 216static u64 oa_report_id(struct xe_oa_stream *stream, void *report) 217{ 218 return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report; 219} 220 221static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report) 222{ 223 if (oa_report_header_64bit(stream)) 224 *(u64 *)report = 0; 225 else 226 *report = 0; 227} 228 229static u64 oa_timestamp(struct xe_oa_stream *stream, void *report) 230{ 231 return oa_report_header_64bit(stream) ? 232 *((u64 *)report + 1) : 233 *((u32 *)report + 1); 234} 235 236static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report) 237{ 238 if (oa_report_header_64bit(stream)) 239 *(u64 *)&report[2] = 0; 240 else 241 report[1] = 0; 242} 243 244static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream) 245{ 246 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); 247 u32 tail, hw_tail, partial_report_size, available; 248 int report_size = stream->oa_buffer.format->size; 249 unsigned long flags; 250 251 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); 252 253 hw_tail = xe_oa_hw_tail_read(stream); 254 hw_tail -= gtt_offset; 255 256 /* 257 * The tail pointer increases in 64 byte (cacheline size), not in report_size 258 * increments. Also report size may not be a power of 2. Compute potential 259 * partially landed report in OA buffer. 260 */ 261 partial_report_size = xe_oa_circ_diff(stream, hw_tail, stream->oa_buffer.tail); 262 partial_report_size %= report_size; 263 264 /* Subtract partial amount off the tail */ 265 hw_tail = xe_oa_circ_diff(stream, hw_tail, partial_report_size); 266 267 tail = hw_tail; 268 269 /* 270 * Walk the stream backward until we find a report with report id and timestamp 271 * not 0. We can't tell whether a report has fully landed in memory before the 272 * report id and timestamp of the following report have landed. 273 * 274 * This is assuming that the writes of the OA unit land in memory in the order 275 * they were written. If not : (╯°□°）╯︵ ┻━┻ 276 */ 277 while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) { 278 void *report = stream->oa_buffer.vaddr + tail; 279 280 if (oa_report_id(stream, report) || oa_timestamp(stream, report)) 281 break; 282 283 tail = xe_oa_circ_diff(stream, tail, report_size); 284 } 285 286 if (xe_oa_circ_diff(stream, hw_tail, tail) > report_size) 287 drm_dbg(&stream->oa->xe->drm, 288 "unlanded report(s) head=0x%x tail=0x%x hw_tail=0x%x\n", 289 stream->oa_buffer.head, tail, hw_tail); 290 291 stream->oa_buffer.tail = tail; 292 293 available = xe_oa_circ_diff(stream, stream->oa_buffer.tail, stream->oa_buffer.head); 294 stream->pollin = available >= stream->wait_num_reports * report_size; 295 296 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); 297 298 return stream->pollin; 299} 300 301static enum hrtimer_restart xe_oa_poll_check_timer_cb(struct hrtimer *hrtimer) 302{ 303 struct xe_oa_stream *stream = 304 container_of(hrtimer, typeof(*stream), poll_check_timer); 305 306 if (xe_oa_buffer_check_unlocked(stream)) 307 wake_up(&stream->poll_wq); 308 309 hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period_ns)); 310 311 return HRTIMER_RESTART; 312} 313 314static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf, 315 size_t count, size_t *offset, const u8 *report) 316{ 317 int report_size = stream->oa_buffer.format->size; 318 int report_size_partial; 319 u8 *oa_buf_end; 320 321 if ((count - *offset) < report_size) 322 return -ENOSPC; 323 324 buf += *offset; 325 326 oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; 327 report_size_partial = oa_buf_end - report; 328 329 if (report_size_partial < report_size) { 330 if (copy_to_user(buf, report, report_size_partial)) 331 return -EFAULT; 332 buf += report_size_partial; 333 334 if (copy_to_user(buf, stream->oa_buffer.vaddr, 335 report_size - report_size_partial)) 336 return -EFAULT; 337 } else if (copy_to_user(buf, report, report_size)) { 338 return -EFAULT; 339 } 340 341 *offset += report_size; 342 343 return 0; 344} 345 346static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf, 347 size_t count, size_t *offset) 348{ 349 int report_size = stream->oa_buffer.format->size; 350 u8 *oa_buf_base = stream->oa_buffer.vaddr; 351 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); 352 size_t start_offset = *offset; 353 unsigned long flags; 354 u32 head, tail; 355 int ret = 0; 356 357 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); 358 head = stream->oa_buffer.head; 359 tail = stream->oa_buffer.tail; 360 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); 361 362 xe_assert(stream->oa->xe, 363 head < stream->oa_buffer.circ_size && tail < stream->oa_buffer.circ_size); 364 365 for (; xe_oa_circ_diff(stream, tail, head); 366 head = xe_oa_circ_incr(stream, head, report_size)) { 367 u8 *report = oa_buf_base + head; 368 369 ret = xe_oa_append_report(stream, buf, count, offset, report); 370 if (ret) 371 break; 372 373 if (!(stream->oa_buffer.circ_size % report_size)) { 374 /* Clear out report id and timestamp to detect unlanded reports */ 375 oa_report_id_clear(stream, (void *)report); 376 oa_timestamp_clear(stream, (void *)report); 377 } else { 378 u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; 379 u32 part = oa_buf_end - report; 380 381 /* Zero out the entire report */ 382 if (report_size <= part) { 383 memset(report, 0, report_size); 384 } else { 385 memset(report, 0, part); 386 memset(oa_buf_base, 0, report_size - part); 387 } 388 } 389 } 390 391 if (start_offset != *offset) { 392 struct xe_reg oaheadptr = __oa_regs(stream)->oa_head_ptr; 393 394 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); 395 xe_mmio_write32(&stream->gt->mmio, oaheadptr, 396 (head + gtt_offset) & OAG_OAHEADPTR_MASK); 397 stream->oa_buffer.head = head; 398 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); 399 } 400 401 return ret; 402} 403 404static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream) 405{ 406 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); 407 int size_exponent = __ffs(xe_bo_size(stream->oa_buffer.bo)); 408 u32 oa_buf = gtt_offset | OAG_OABUFFER_MEMORY_SELECT; 409 struct xe_mmio *mmio = &stream->gt->mmio; 410 unsigned long flags; 411 412 /* 413 * If oa buffer size is more than 16MB (exponent greater than 24), the 414 * oa buffer size field is multiplied by 8 in xe_oa_enable_metric_set. 415 */ 416 oa_buf |= REG_FIELD_PREP(OABUFFER_SIZE_MASK, 417 size_exponent > 24 ? size_exponent - 20 : size_exponent - 17); 418 419 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); 420 421 xe_mmio_write32(mmio, __oa_regs(stream)->oa_status, 0); 422 xe_mmio_write32(mmio, __oa_regs(stream)->oa_head_ptr, 423 gtt_offset & OAG_OAHEADPTR_MASK); 424 stream->oa_buffer.head = 0; 425 /* 426 * PRM says: "This MMIO must be set before the OATAILPTR register and after the 427 * OAHEADPTR register. This is to enable proper functionality of the overflow bit". 428 */ 429 xe_mmio_write32(mmio, __oa_regs(stream)->oa_buffer, oa_buf); 430 xe_mmio_write32(mmio, __oa_regs(stream)->oa_tail_ptr, 431 gtt_offset & OAG_OATAILPTR_MASK); 432 433 /* Mark that we need updated tail pointer to read from */ 434 stream->oa_buffer.tail = 0; 435 436 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); 437 438 /* Zero out the OA buffer since we rely on zero report id and timestamp fields */ 439 memset(stream->oa_buffer.vaddr, 0, xe_bo_size(stream->oa_buffer.bo)); 440} 441 442static u32 __format_to_oactrl(const struct xe_oa_format *format, int counter_sel_mask) 443{ 444 return ((format->counter_select << (ffs(counter_sel_mask) - 1)) & counter_sel_mask) | 445 REG_FIELD_PREP(OA_OACONTROL_REPORT_BC_MASK, format->bc_report) | 446 REG_FIELD_PREP(OA_OACONTROL_COUNTER_SIZE_MASK, format->counter_size); 447} 448 449static u32 __oa_ccs_select(struct xe_oa_stream *stream) 450{ 451 u32 val; 452 453 if (stream->hwe->class != XE_ENGINE_CLASS_COMPUTE) 454 return 0; 455 456 val = REG_FIELD_PREP(OAG_OACONTROL_OA_CCS_SELECT_MASK, stream->hwe->instance); 457 xe_assert(stream->oa->xe, 458 REG_FIELD_GET(OAG_OACONTROL_OA_CCS_SELECT_MASK, val) == stream->hwe->instance); 459 return val; 460} 461 462static u32 __oactrl_used_bits(struct xe_oa_stream *stream) 463{ 464 return stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG ? 465 OAG_OACONTROL_USED_BITS : OAM_OACONTROL_USED_BITS; 466} 467 468static void xe_oa_enable(struct xe_oa_stream *stream) 469{ 470 const struct xe_oa_format *format = stream->oa_buffer.format; 471 const struct xe_oa_regs *regs; 472 u32 val; 473 474 /* 475 * BSpec: 46822: Bit 0. Even if stream->sample is 0, for OAR to function, the OA 476 * buffer must be correctly initialized 477 */ 478 xe_oa_init_oa_buffer(stream); 479 480 regs = __oa_regs(stream); 481 val = __format_to_oactrl(format, regs->oa_ctrl_counter_select_mask) | 482 __oa_ccs_select(stream) | OAG_OACONTROL_OA_COUNTER_ENABLE; 483 484 if (GRAPHICS_VER(stream->oa->xe) >= 20 && 485 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG) 486 val |= OAG_OACONTROL_OA_PES_DISAG_EN; 487 488 xe_mmio_rmw32(&stream->gt->mmio, regs->oa_ctrl, __oactrl_used_bits(stream), val); 489} 490 491static void xe_oa_disable(struct xe_oa_stream *stream) 492{ 493 struct xe_mmio *mmio = &stream->gt->mmio; 494 495 xe_mmio_rmw32(mmio, __oa_regs(stream)->oa_ctrl, __oactrl_used_bits(stream), 0); 496 if (xe_mmio_wait32(mmio, __oa_regs(stream)->oa_ctrl, 497 OAG_OACONTROL_OA_COUNTER_ENABLE, 0, 50000, NULL, false)) 498 drm_err(&stream->oa->xe->drm, 499 "wait for OA to be disabled timed out\n"); 500 501 if (GRAPHICS_VERx100(stream->oa->xe) <= 1270 && GRAPHICS_VERx100(stream->oa->xe) != 1260) { 502 /* <= XE_METEORLAKE except XE_PVC */ 503 xe_mmio_write32(mmio, OA_TLB_INV_CR, 1); 504 if (xe_mmio_wait32(mmio, OA_TLB_INV_CR, 1, 0, 50000, NULL, false)) 505 drm_err(&stream->oa->xe->drm, 506 "wait for OA tlb invalidate timed out\n"); 507 } 508} 509 510static int xe_oa_wait_unlocked(struct xe_oa_stream *stream) 511{ 512 /* We might wait indefinitely if periodic sampling is not enabled */ 513 if (!stream->periodic) 514 return -EINVAL; 515 516 return wait_event_interruptible(stream->poll_wq, 517 xe_oa_buffer_check_unlocked(stream)); 518} 519 520#define OASTATUS_RELEVANT_BITS (OASTATUS_MMIO_TRG_Q_FULL | OASTATUS_COUNTER_OVERFLOW | \ 521 OASTATUS_BUFFER_OVERFLOW | OASTATUS_REPORT_LOST) 522 523static int __xe_oa_read(struct xe_oa_stream *stream, char __user *buf, 524 size_t count, size_t *offset) 525{ 526 /* Only clear our bits to avoid side-effects */ 527 stream->oa_status = xe_mmio_rmw32(&stream->gt->mmio, __oa_regs(stream)->oa_status, 528 OASTATUS_RELEVANT_BITS, 0); 529 /* 530 * Signal to userspace that there is non-zero OA status to read via 531 * @DRM_XE_OBSERVATION_IOCTL_STATUS observation stream fd ioctl 532 */ 533 if (stream->oa_status & OASTATUS_RELEVANT_BITS) 534 return -EIO; 535 536 return xe_oa_append_reports(stream, buf, count, offset); 537} 538 539static ssize_t xe_oa_read(struct file *file, char __user *buf, 540 size_t count, loff_t *ppos) 541{ 542 struct xe_oa_stream *stream = file->private_data; 543 size_t offset = 0; 544 int ret; 545 546 if (!stream->sample) 547 return -EINVAL; 548 549 if (!(file->f_flags & O_NONBLOCK)) { 550 do { 551 ret = xe_oa_wait_unlocked(stream); 552 if (ret) 553 return ret; 554 555 mutex_lock(&stream->stream_lock); 556 ret = __xe_oa_read(stream, buf, count, &offset); 557 mutex_unlock(&stream->stream_lock); 558 } while (!offset && !ret); 559 } else { 560 xe_oa_buffer_check_unlocked(stream); 561 mutex_lock(&stream->stream_lock); 562 ret = __xe_oa_read(stream, buf, count, &offset); 563 mutex_unlock(&stream->stream_lock); 564 } 565 566 /* 567 * Typically we clear pollin here in order to wait for the new hrtimer callback 568 * before unblocking. The exception to this is if __xe_oa_read returns -ENOSPC, 569 * which means that more OA data is available than could fit in the user provided 570 * buffer. In this case we want the next poll() call to not block. 571 * 572 * Also in case of -EIO, we have already waited for data before returning 573 * -EIO, so need to wait again 574 */ 575 if (ret != -ENOSPC && ret != -EIO) 576 stream->pollin = false; 577 578 /* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, -EINVAL, ... */ 579 return offset ?: (ret ?: -EAGAIN); 580} 581 582static __poll_t xe_oa_poll_locked(struct xe_oa_stream *stream, 583 struct file *file, poll_table *wait) 584{ 585 __poll_t events = 0; 586 587 poll_wait(file, &stream->poll_wq, wait); 588 589 /* 590 * We don't explicitly check whether there's something to read here since this 591 * path may be hot depending on what else userspace is polling, or on the timeout 592 * in use. We rely on hrtimer xe_oa_poll_check_timer_cb to notify us when there 593 * are samples to read 594 */ 595 if (stream->pollin) 596 events |= EPOLLIN; 597 598 return events; 599} 600 601static __poll_t xe_oa_poll(struct file *file, poll_table *wait) 602{ 603 struct xe_oa_stream *stream = file->private_data; 604 __poll_t ret; 605 606 mutex_lock(&stream->stream_lock); 607 ret = xe_oa_poll_locked(stream, file, wait); 608 mutex_unlock(&stream->stream_lock); 609 610 return ret; 611} 612 613static void xe_oa_lock_vma(struct xe_exec_queue *q) 614{ 615 if (q->vm) { 616 down_read(&q->vm->lock); 617 xe_vm_lock(q->vm, false); 618 } 619} 620 621static void xe_oa_unlock_vma(struct xe_exec_queue *q) 622{ 623 if (q->vm) { 624 xe_vm_unlock(q->vm); 625 up_read(&q->vm->lock); 626 } 627} 628 629static struct dma_fence *xe_oa_submit_bb(struct xe_oa_stream *stream, enum xe_oa_submit_deps deps, 630 struct xe_bb *bb) 631{ 632 struct xe_exec_queue *q = stream->exec_q ?: stream->k_exec_q; 633 struct xe_sched_job *job; 634 struct dma_fence *fence; 635 int err = 0; 636 637 xe_oa_lock_vma(q); 638 639 job = xe_bb_create_job(q, bb); 640 if (IS_ERR(job)) { 641 err = PTR_ERR(job); 642 goto exit; 643 } 644 job->ggtt = true; 645 646 if (deps == XE_OA_SUBMIT_ADD_DEPS) { 647 for (int i = 0; i < stream->num_syncs && !err; i++) 648 err = xe_sync_entry_add_deps(&stream->syncs[i], job); 649 if (err) { 650 drm_dbg(&stream->oa->xe->drm, "xe_sync_entry_add_deps err %d\n", err); 651 goto err_put_job; 652 } 653 } 654 655 xe_sched_job_arm(job); 656 fence = dma_fence_get(&job->drm.s_fence->finished); 657 xe_sched_job_push(job); 658 659 xe_oa_unlock_vma(q); 660 661 return fence; 662err_put_job: 663 xe_sched_job_put(job); 664exit: 665 xe_oa_unlock_vma(q); 666 return ERR_PTR(err); 667} 668 669static void write_cs_mi_lri(struct xe_bb *bb, const struct xe_oa_reg *reg_data, u32 n_regs) 670{ 671 u32 i; 672 673#define MI_LOAD_REGISTER_IMM_MAX_REGS (126) 674 675 for (i = 0; i < n_regs; i++) { 676 if ((i % MI_LOAD_REGISTER_IMM_MAX_REGS) == 0) { 677 u32 n_lri = min_t(u32, n_regs - i, 678 MI_LOAD_REGISTER_IMM_MAX_REGS); 679 680 bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(n_lri); 681 } 682 bb->cs[bb->len++] = reg_data[i].addr.addr; 683 bb->cs[bb->len++] = reg_data[i].value; 684 } 685} 686 687static int num_lri_dwords(int num_regs) 688{ 689 int count = 0; 690 691 if (num_regs > 0) { 692 count += DIV_ROUND_UP(num_regs, MI_LOAD_REGISTER_IMM_MAX_REGS); 693 count += num_regs * 2; 694 } 695 696 return count; 697} 698 699static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream) 700{ 701 xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); 702} 703 704static void xe_oa_free_configs(struct xe_oa_stream *stream) 705{ 706 struct xe_oa_config_bo *oa_bo, *tmp; 707 708 xe_oa_config_put(stream->oa_config); 709 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) 710 free_oa_config_bo(oa_bo, stream->last_fence); 711 dma_fence_put(stream->last_fence); 712} 713 714static int xe_oa_load_with_lri(struct xe_oa_stream *stream, struct xe_oa_reg *reg_lri, u32 count) 715{ 716 struct dma_fence *fence; 717 struct xe_bb *bb; 718 int err; 719 720 bb = xe_bb_new(stream->gt, 2 * count + 1, false); 721 if (IS_ERR(bb)) { 722 err = PTR_ERR(bb); 723 goto exit; 724 } 725 726 write_cs_mi_lri(bb, reg_lri, count); 727 728 fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_NO_DEPS, bb); 729 if (IS_ERR(fence)) { 730 err = PTR_ERR(fence); 731 goto free_bb; 732 } 733 xe_bb_free(bb, fence); 734 dma_fence_put(fence); 735 736 return 0; 737free_bb: 738 xe_bb_free(bb, NULL); 739exit: 740 return err; 741} 742 743static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable) 744{ 745 const struct xe_oa_format *format = stream->oa_buffer.format; 746 u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) | 747 (enable ? OAR_OACONTROL_COUNTER_ENABLE : 0); 748 749 struct xe_oa_reg reg_lri[] = { 750 { 751 OACTXCONTROL(stream->hwe->mmio_base), 752 enable ? OA_COUNTER_RESUME : 0, 753 }, 754 { 755 OAR_OACONTROL, 756 oacontrol, 757 }, 758 { 759 RING_CONTEXT_CONTROL(stream->hwe->mmio_base), 760 _MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE, 761 enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0) 762 }, 763 }; 764 765 return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri)); 766} 767 768static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable) 769{ 770 const struct xe_oa_format *format = stream->oa_buffer.format; 771 u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) | 772 (enable ? OAR_OACONTROL_COUNTER_ENABLE : 0); 773 struct xe_oa_reg reg_lri[] = { 774 { 775 OACTXCONTROL(stream->hwe->mmio_base), 776 enable ? OA_COUNTER_RESUME : 0, 777 }, 778 { 779 OAC_OACONTROL, 780 oacontrol 781 }, 782 { 783 RING_CONTEXT_CONTROL(stream->hwe->mmio_base), 784 _MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE, 785 enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0) | 786 _MASKED_FIELD(CTX_CTRL_RUN_ALONE, enable ? CTX_CTRL_RUN_ALONE : 0), 787 }, 788 }; 789 790 /* Set ccs select to enable programming of OAC_OACONTROL */ 791 xe_mmio_write32(&stream->gt->mmio, __oa_regs(stream)->oa_ctrl, 792 __oa_ccs_select(stream)); 793 794 return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri)); 795} 796 797static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable) 798{ 799 switch (stream->hwe->class) { 800 case XE_ENGINE_CLASS_RENDER: 801 return xe_oa_configure_oar_context(stream, enable); 802 case XE_ENGINE_CLASS_COMPUTE: 803 return xe_oa_configure_oac_context(stream, enable); 804 default: 805 /* Video engines do not support MI_REPORT_PERF_COUNT */ 806 return 0; 807 } 808} 809 810#define HAS_OA_BPC_REPORTING(xe) (GRAPHICS_VERx100(xe) >= 1255) 811 812static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable) 813{ 814 return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_MMIO_TRG, 815 enable && stream && stream->sample ? 816 0 : OAG_OA_DEBUG_DISABLE_MMIO_TRG); 817} 818 819static void xe_oa_disable_metric_set(struct xe_oa_stream *stream) 820{ 821 struct xe_mmio *mmio = &stream->gt->mmio; 822 u32 sqcnt1; 823 824 /* Enable thread stall DOP gating and EU DOP gating. */ 825 if (XE_GT_WA(stream->gt, 1508761755)) { 826 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, 827 _MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE)); 828 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, 829 _MASKED_BIT_DISABLE(DISABLE_DOP_GATING)); 830 } 831 832 xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug, 833 oag_configure_mmio_trigger(stream, false)); 834 835 /* disable the context save/restore or OAR counters */ 836 if (stream->exec_q) 837 xe_oa_configure_oa_context(stream, false); 838 839 /* Make sure we disable noa to save power. */ 840 if (GT_VER(stream->gt) < 35) 841 xe_mmio_rmw32(mmio, RPM_CONFIG1, GT_NOA_ENABLE, 0); 842 843 sqcnt1 = SQCNT1_PMON_ENABLE | 844 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); 845 846 /* Reset PMON Enable to save power. */ 847 xe_mmio_rmw32(mmio, XELPMP_SQCNT1, sqcnt1, 0); 848 849 if ((stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM || 850 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM_SAG) && 851 GRAPHICS_VER(stream->oa->xe) >= 30) 852 xe_mmio_rmw32(mmio, OAM_COMPRESSION_T3_CONTROL, OAM_LAT_MEASURE_ENABLE, 0); 853} 854 855static void xe_oa_stream_destroy(struct xe_oa_stream *stream) 856{ 857 struct xe_oa_unit *u = stream->oa_unit; 858 struct xe_gt *gt = stream->hwe->gt; 859 860 if (WARN_ON(stream != u->exclusive_stream)) 861 return; 862 863 WRITE_ONCE(u->exclusive_stream, NULL); 864 865 mutex_destroy(&stream->stream_lock); 866 867 xe_oa_disable_metric_set(stream); 868 xe_exec_queue_put(stream->k_exec_q); 869 870 xe_oa_free_oa_buffer(stream); 871 872 xe_force_wake_put(gt_to_fw(gt), stream->fw_ref); 873 xe_pm_runtime_put(stream->oa->xe); 874 875 /* Wa_1509372804:pvc: Unset the override of GUCRC mode to enable rc6 */ 876 if (stream->override_gucrc) 877 xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(>->uc.guc.pc)); 878 879 xe_oa_free_configs(stream); 880 xe_file_put(stream->xef); 881} 882 883static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream, size_t size) 884{ 885 struct xe_bo *bo; 886 887 bo = xe_bo_create_pin_map_novm(stream->oa->xe, stream->gt->tile, 888 size, ttm_bo_type_kernel, 889 XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT, false); 890 if (IS_ERR(bo)) 891 return PTR_ERR(bo); 892 893 stream->oa_buffer.bo = bo; 894 /* mmap implementation requires OA buffer to be in system memory */ 895 xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0); 896 stream->oa_buffer.vaddr = bo->vmap.vaddr; 897 return 0; 898} 899 900static struct xe_oa_config_bo * 901__xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config) 902{ 903 struct xe_oa_config_bo *oa_bo; 904 size_t config_length; 905 struct xe_bb *bb; 906 907 oa_bo = kzalloc(sizeof(*oa_bo), GFP_KERNEL); 908 if (!oa_bo) 909 return ERR_PTR(-ENOMEM); 910 911 config_length = num_lri_dwords(oa_config->regs_len); 912 config_length = ALIGN(sizeof(u32) * config_length, XE_PAGE_SIZE) / sizeof(u32); 913 914 bb = xe_bb_new(stream->gt, config_length, false); 915 if (IS_ERR(bb)) 916 goto err_free; 917 918 write_cs_mi_lri(bb, oa_config->regs, oa_config->regs_len); 919 920 oa_bo->bb = bb; 921 oa_bo->oa_config = xe_oa_config_get(oa_config); 922 llist_add(&oa_bo->node, &stream->oa_config_bos); 923 924 return oa_bo; 925err_free: 926 kfree(oa_bo); 927 return ERR_CAST(bb); 928} 929 930static struct xe_oa_config_bo * 931xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config) 932{ 933 struct xe_oa_config_bo *oa_bo; 934 935 /* Look for the buffer in the already allocated BOs attached to the stream */ 936 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { 937 if (oa_bo->oa_config == oa_config && 938 memcmp(oa_bo->oa_config->uuid, oa_config->uuid, 939 sizeof(oa_config->uuid)) == 0) 940 goto out; 941 } 942 943 oa_bo = __xe_oa_alloc_config_buffer(stream, oa_config); 944out: 945 return oa_bo; 946} 947 948static void xe_oa_update_last_fence(struct xe_oa_stream *stream, struct dma_fence *fence) 949{ 950 dma_fence_put(stream->last_fence); 951 stream->last_fence = dma_fence_get(fence); 952} 953 954static void xe_oa_fence_work_fn(struct work_struct *w) 955{ 956 struct xe_oa_fence *ofence = container_of(w, typeof(*ofence), work.work); 957 958 /* Signal fence to indicate new OA configuration is active */ 959 dma_fence_signal(&ofence->base); 960 dma_fence_put(&ofence->base); 961} 962 963static void xe_oa_config_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 964{ 965 /* Additional empirical delay needed for NOA programming after registers are written */ 966#define NOA_PROGRAM_ADDITIONAL_DELAY_US 500 967 968 struct xe_oa_fence *ofence = container_of(cb, typeof(*ofence), cb); 969 970 INIT_DELAYED_WORK(&ofence->work, xe_oa_fence_work_fn); 971 queue_delayed_work(system_unbound_wq, &ofence->work, 972 usecs_to_jiffies(NOA_PROGRAM_ADDITIONAL_DELAY_US)); 973 dma_fence_put(fence); 974} 975 976static const char *xe_oa_get_driver_name(struct dma_fence *fence) 977{ 978 return "xe_oa"; 979} 980 981static const char *xe_oa_get_timeline_name(struct dma_fence *fence) 982{ 983 return "unbound"; 984} 985 986static const struct dma_fence_ops xe_oa_fence_ops = { 987 .get_driver_name = xe_oa_get_driver_name, 988 .get_timeline_name = xe_oa_get_timeline_name, 989}; 990 991static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config *config) 992{ 993#define NOA_PROGRAM_ADDITIONAL_DELAY_US 500 994 struct xe_oa_config_bo *oa_bo; 995 struct xe_oa_fence *ofence; 996 int i, err, num_signal = 0; 997 struct dma_fence *fence; 998 999 ofence = kzalloc(sizeof(*ofence), GFP_KERNEL); 1000 if (!ofence) { 1001 err = -ENOMEM; 1002 goto exit; 1003 } 1004 1005 oa_bo = xe_oa_alloc_config_buffer(stream, config); 1006 if (IS_ERR(oa_bo)) { 1007 err = PTR_ERR(oa_bo); 1008 goto exit; 1009 } 1010 1011 /* Emit OA configuration batch */ 1012 fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_ADD_DEPS, oa_bo->bb); 1013 if (IS_ERR(fence)) { 1014 err = PTR_ERR(fence); 1015 goto exit; 1016 } 1017 1018 /* Point of no return: initialize and set fence to signal */ 1019 spin_lock_init(&ofence->lock); 1020 dma_fence_init(&ofence->base, &xe_oa_fence_ops, &ofence->lock, 0, 0); 1021 1022 for (i = 0; i < stream->num_syncs; i++) { 1023 if (stream->syncs[i].flags & DRM_XE_SYNC_FLAG_SIGNAL) 1024 num_signal++; 1025 xe_sync_entry_signal(&stream->syncs[i], &ofence->base); 1026 } 1027 1028 /* Additional dma_fence_get in case we dma_fence_wait */ 1029 if (!num_signal) 1030 dma_fence_get(&ofence->base); 1031 1032 /* Update last fence too before adding callback */ 1033 xe_oa_update_last_fence(stream, fence); 1034 1035 /* Add job fence callback to schedule work to signal ofence->base */ 1036 err = dma_fence_add_callback(fence, &ofence->cb, xe_oa_config_cb); 1037 xe_gt_assert(stream->gt, !err || err == -ENOENT); 1038 if (err == -ENOENT) 1039 xe_oa_config_cb(fence, &ofence->cb); 1040 1041 /* If nothing needs to be signaled we wait synchronously */ 1042 if (!num_signal) { 1043 dma_fence_wait(&ofence->base, false); 1044 dma_fence_put(&ofence->base); 1045 } 1046 1047 /* Done with syncs */ 1048 for (i = 0; i < stream->num_syncs; i++) 1049 xe_sync_entry_cleanup(&stream->syncs[i]); 1050 kfree(stream->syncs); 1051 1052 return 0; 1053exit: 1054 kfree(ofence); 1055 return err; 1056} 1057 1058static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream) 1059{ 1060 /* If user didn't require OA reports, ask HW not to emit ctx switch reports */ 1061 return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS, 1062 stream->sample ? 1063 0 : OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS); 1064} 1065 1066static u32 oag_buf_size_select(const struct xe_oa_stream *stream) 1067{ 1068 return _MASKED_FIELD(OAG_OA_DEBUG_BUF_SIZE_SELECT, 1069 xe_bo_size(stream->oa_buffer.bo) > SZ_16M ? 1070 OAG_OA_DEBUG_BUF_SIZE_SELECT : 0); 1071} 1072 1073static int xe_oa_enable_metric_set(struct xe_oa_stream *stream) 1074{ 1075 struct xe_mmio *mmio = &stream->gt->mmio; 1076 u32 oa_debug, sqcnt1; 1077 int ret; 1078 1079 /* 1080 * EU NOA signals behave incorrectly if EU clock gating is enabled. 1081 * Disable thread stall DOP gating and EU DOP gating. 1082 */ 1083 if (XE_GT_WA(stream->gt, 1508761755)) { 1084 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, 1085 _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE)); 1086 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, 1087 _MASKED_BIT_ENABLE(DISABLE_DOP_GATING)); 1088 } 1089 1090 /* Disable clk ratio reports */ 1091 oa_debug = OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS | 1092 OAG_OA_DEBUG_INCLUDE_CLK_RATIO; 1093 1094 if (GRAPHICS_VER(stream->oa->xe) >= 20) 1095 oa_debug |= 1096 /* The three bits below are needed to get PEC counters running */ 1097 OAG_OA_DEBUG_START_TRIGGER_SCOPE_CONTROL | 1098 OAG_OA_DEBUG_DISABLE_START_TRG_2_COUNT_QUAL | 1099 OAG_OA_DEBUG_DISABLE_START_TRG_1_COUNT_QUAL; 1100 1101 xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug, 1102 _MASKED_BIT_ENABLE(oa_debug) | 1103 oag_report_ctx_switches(stream) | 1104 oag_buf_size_select(stream) | 1105 oag_configure_mmio_trigger(stream, true)); 1106 1107 xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, 1108 OAG_OAGLBCTXCTRL_COUNTER_RESUME | 1109 (stream->periodic ? 1110 OAG_OAGLBCTXCTRL_TIMER_ENABLE | 1111 REG_FIELD_PREP(OAG_OAGLBCTXCTRL_TIMER_PERIOD_MASK, 1112 stream->period_exponent) : 0)); 1113 1114 /* 1115 * Initialize Super Queue Internal Cnt Register 1116 * Set PMON Enable in order to collect valid metrics 1117 * Enable bytes per clock reporting 1118 */ 1119 sqcnt1 = SQCNT1_PMON_ENABLE | 1120 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); 1121 xe_mmio_rmw32(mmio, XELPMP_SQCNT1, 0, sqcnt1); 1122 1123 if ((stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM || 1124 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM_SAG) && 1125 GRAPHICS_VER(stream->oa->xe) >= 30) 1126 xe_mmio_rmw32(mmio, OAM_COMPRESSION_T3_CONTROL, 0, OAM_LAT_MEASURE_ENABLE); 1127 1128 /* Configure OAR/OAC */ 1129 if (stream->exec_q) { 1130 ret = xe_oa_configure_oa_context(stream, true); 1131 if (ret) 1132 return ret; 1133 } 1134 1135 return xe_oa_emit_oa_config(stream, stream->oa_config); 1136} 1137 1138static int decode_oa_format(struct xe_oa *oa, u64 fmt, enum xe_oa_format_name *name) 1139{ 1140 u32 counter_size = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SIZE, fmt); 1141 u32 counter_sel = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SEL, fmt); 1142 u32 bc_report = FIELD_GET(DRM_XE_OA_FORMAT_MASK_BC_REPORT, fmt); 1143 u32 type = FIELD_GET(DRM_XE_OA_FORMAT_MASK_FMT_TYPE, fmt); 1144 int idx; 1145 1146 for_each_set_bit(idx, oa->format_mask, __XE_OA_FORMAT_MAX) { 1147 const struct xe_oa_format *f = &oa->oa_formats[idx]; 1148 1149 if (counter_size == f->counter_size && bc_report == f->bc_report && 1150 type == f->type && counter_sel == f->counter_select) { 1151 *name = idx; 1152 return 0; 1153 } 1154 } 1155 1156 return -EINVAL; 1157} 1158 1159static struct xe_oa_unit *xe_oa_lookup_oa_unit(struct xe_oa *oa, u32 oa_unit_id) 1160{ 1161 struct xe_gt *gt; 1162 int gt_id, i; 1163 1164 for_each_gt(gt, oa->xe, gt_id) { 1165 for (i = 0; i < gt->oa.num_oa_units; i++) { 1166 struct xe_oa_unit *u = >->oa.oa_unit[i]; 1167 1168 if (u->oa_unit_id == oa_unit_id) 1169 return u; 1170 } 1171 } 1172 1173 return NULL; 1174} 1175 1176static int xe_oa_set_prop_oa_unit_id(struct xe_oa *oa, u64 value, 1177 struct xe_oa_open_param *param) 1178{ 1179 param->oa_unit = xe_oa_lookup_oa_unit(oa, value); 1180 if (!param->oa_unit) { 1181 drm_dbg(&oa->xe->drm, "OA unit ID out of range %lld\n", value); 1182 return -EINVAL; 1183 } 1184 return 0; 1185} 1186 1187static int xe_oa_set_prop_sample_oa(struct xe_oa *oa, u64 value, 1188 struct xe_oa_open_param *param) 1189{ 1190 param->sample = value; 1191 return 0; 1192} 1193 1194static int xe_oa_set_prop_metric_set(struct xe_oa *oa, u64 value, 1195 struct xe_oa_open_param *param) 1196{ 1197 param->metric_set = value; 1198 return 0; 1199} 1200 1201static int xe_oa_set_prop_oa_format(struct xe_oa *oa, u64 value, 1202 struct xe_oa_open_param *param) 1203{ 1204 int ret = decode_oa_format(oa, value, ¶m->oa_format); 1205 1206 if (ret) { 1207 drm_dbg(&oa->xe->drm, "Unsupported OA report format %#llx\n", value); 1208 return ret; 1209 } 1210 return 0; 1211} 1212 1213static int xe_oa_set_prop_oa_exponent(struct xe_oa *oa, u64 value, 1214 struct xe_oa_open_param *param) 1215{ 1216#define OA_EXPONENT_MAX 31 1217 1218 if (value > OA_EXPONENT_MAX) { 1219 drm_dbg(&oa->xe->drm, "OA timer exponent too high (> %u)\n", OA_EXPONENT_MAX); 1220 return -EINVAL; 1221 } 1222 param->period_exponent = value; 1223 return 0; 1224} 1225 1226static int xe_oa_set_prop_disabled(struct xe_oa *oa, u64 value, 1227 struct xe_oa_open_param *param) 1228{ 1229 param->disabled = value; 1230 return 0; 1231} 1232 1233static int xe_oa_set_prop_exec_queue_id(struct xe_oa *oa, u64 value, 1234 struct xe_oa_open_param *param) 1235{ 1236 param->exec_queue_id = value; 1237 return 0; 1238} 1239 1240static int xe_oa_set_prop_engine_instance(struct xe_oa *oa, u64 value, 1241 struct xe_oa_open_param *param) 1242{ 1243 param->engine_instance = value; 1244 return 0; 1245} 1246 1247static int xe_oa_set_no_preempt(struct xe_oa *oa, u64 value, 1248 struct xe_oa_open_param *param) 1249{ 1250 param->no_preempt = value; 1251 return 0; 1252} 1253 1254static int xe_oa_set_prop_num_syncs(struct xe_oa *oa, u64 value, 1255 struct xe_oa_open_param *param) 1256{ 1257 if (XE_IOCTL_DBG(oa->xe, value > DRM_XE_MAX_SYNCS)) 1258 return -EINVAL; 1259 1260 param->num_syncs = value; 1261 return 0; 1262} 1263 1264static int xe_oa_set_prop_syncs_user(struct xe_oa *oa, u64 value, 1265 struct xe_oa_open_param *param) 1266{ 1267 param->syncs_user = u64_to_user_ptr(value); 1268 return 0; 1269} 1270 1271static int xe_oa_set_prop_oa_buffer_size(struct xe_oa *oa, u64 value, 1272 struct xe_oa_open_param *param) 1273{ 1274 if (!is_power_of_2(value) || value < SZ_128K || value > SZ_128M) { 1275 drm_dbg(&oa->xe->drm, "OA buffer size invalid %llu\n", value); 1276 return -EINVAL; 1277 } 1278 param->oa_buffer_size = value; 1279 return 0; 1280} 1281 1282static int xe_oa_set_prop_wait_num_reports(struct xe_oa *oa, u64 value, 1283 struct xe_oa_open_param *param) 1284{ 1285 if (!value) { 1286 drm_dbg(&oa->xe->drm, "wait_num_reports %llu\n", value); 1287 return -EINVAL; 1288 } 1289 param->wait_num_reports = value; 1290 return 0; 1291} 1292 1293static int xe_oa_set_prop_ret_inval(struct xe_oa *oa, u64 value, 1294 struct xe_oa_open_param *param) 1295{ 1296 return -EINVAL; 1297} 1298 1299typedef int (*xe_oa_set_property_fn)(struct xe_oa *oa, u64 value, 1300 struct xe_oa_open_param *param); 1301static const xe_oa_set_property_fn xe_oa_set_property_funcs_open[] = { 1302 [DRM_XE_OA_PROPERTY_OA_UNIT_ID] = xe_oa_set_prop_oa_unit_id, 1303 [DRM_XE_OA_PROPERTY_SAMPLE_OA] = xe_oa_set_prop_sample_oa, 1304 [DRM_XE_OA_PROPERTY_OA_METRIC_SET] = xe_oa_set_prop_metric_set, 1305 [DRM_XE_OA_PROPERTY_OA_FORMAT] = xe_oa_set_prop_oa_format, 1306 [DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT] = xe_oa_set_prop_oa_exponent, 1307 [DRM_XE_OA_PROPERTY_OA_DISABLED] = xe_oa_set_prop_disabled, 1308 [DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID] = xe_oa_set_prop_exec_queue_id, 1309 [DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE] = xe_oa_set_prop_engine_instance, 1310 [DRM_XE_OA_PROPERTY_NO_PREEMPT] = xe_oa_set_no_preempt, 1311 [DRM_XE_OA_PROPERTY_NUM_SYNCS] = xe_oa_set_prop_num_syncs, 1312 [DRM_XE_OA_PROPERTY_SYNCS] = xe_oa_set_prop_syncs_user, 1313 [DRM_XE_OA_PROPERTY_OA_BUFFER_SIZE] = xe_oa_set_prop_oa_buffer_size, 1314 [DRM_XE_OA_PROPERTY_WAIT_NUM_REPORTS] = xe_oa_set_prop_wait_num_reports, 1315}; 1316 1317static const xe_oa_set_property_fn xe_oa_set_property_funcs_config[] = { 1318 [DRM_XE_OA_PROPERTY_OA_UNIT_ID] = xe_oa_set_prop_ret_inval, 1319 [DRM_XE_OA_PROPERTY_SAMPLE_OA] = xe_oa_set_prop_ret_inval, 1320 [DRM_XE_OA_PROPERTY_OA_METRIC_SET] = xe_oa_set_prop_metric_set, 1321 [DRM_XE_OA_PROPERTY_OA_FORMAT] = xe_oa_set_prop_ret_inval, 1322 [DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT] = xe_oa_set_prop_ret_inval, 1323 [DRM_XE_OA_PROPERTY_OA_DISABLED] = xe_oa_set_prop_ret_inval, 1324 [DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID] = xe_oa_set_prop_ret_inval, 1325 [DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE] = xe_oa_set_prop_ret_inval, 1326 [DRM_XE_OA_PROPERTY_NO_PREEMPT] = xe_oa_set_prop_ret_inval, 1327 [DRM_XE_OA_PROPERTY_NUM_SYNCS] = xe_oa_set_prop_num_syncs, 1328 [DRM_XE_OA_PROPERTY_SYNCS] = xe_oa_set_prop_syncs_user, 1329 [DRM_XE_OA_PROPERTY_OA_BUFFER_SIZE] = xe_oa_set_prop_ret_inval, 1330 [DRM_XE_OA_PROPERTY_WAIT_NUM_REPORTS] = xe_oa_set_prop_ret_inval, 1331}; 1332 1333static int xe_oa_user_ext_set_property(struct xe_oa *oa, enum xe_oa_user_extn_from from, 1334 u64 extension, struct xe_oa_open_param *param) 1335{ 1336 u64 __user *address = u64_to_user_ptr(extension); 1337 struct drm_xe_ext_set_property ext; 1338 int err; 1339 u32 idx; 1340 1341 err = copy_from_user(&ext, address, sizeof(ext)); 1342 if (XE_IOCTL_DBG(oa->xe, err)) 1343 return -EFAULT; 1344 1345 BUILD_BUG_ON(ARRAY_SIZE(xe_oa_set_property_funcs_open) != 1346 ARRAY_SIZE(xe_oa_set_property_funcs_config)); 1347 1348 if (XE_IOCTL_DBG(oa->xe, ext.property >= ARRAY_SIZE(xe_oa_set_property_funcs_open)) || 1349 XE_IOCTL_DBG(oa->xe, !ext.property) || XE_IOCTL_DBG(oa->xe, ext.pad)) 1350 return -EINVAL; 1351 1352 idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_oa_set_property_funcs_open)); 1353 1354 if (from == XE_OA_USER_EXTN_FROM_CONFIG) 1355 return xe_oa_set_property_funcs_config[idx](oa, ext.value, param); 1356 else 1357 return xe_oa_set_property_funcs_open[idx](oa, ext.value, param); 1358} 1359 1360typedef int (*xe_oa_user_extension_fn)(struct xe_oa *oa, enum xe_oa_user_extn_from from, 1361 u64 extension, struct xe_oa_open_param *param); 1362static const xe_oa_user_extension_fn xe_oa_user_extension_funcs[] = { 1363 [DRM_XE_OA_EXTENSION_SET_PROPERTY] = xe_oa_user_ext_set_property, 1364}; 1365 1366#define MAX_USER_EXTENSIONS 16 1367static int xe_oa_user_extensions(struct xe_oa *oa, enum xe_oa_user_extn_from from, u64 extension, 1368 int ext_number, struct xe_oa_open_param *param) 1369{ 1370 u64 __user *address = u64_to_user_ptr(extension); 1371 struct drm_xe_user_extension ext; 1372 int err; 1373 u32 idx; 1374 1375 if (XE_IOCTL_DBG(oa->xe, ext_number >= MAX_USER_EXTENSIONS)) 1376 return -E2BIG; 1377 1378 err = copy_from_user(&ext, address, sizeof(ext)); 1379 if (XE_IOCTL_DBG(oa->xe, err)) 1380 return -EFAULT; 1381 1382 if (XE_IOCTL_DBG(oa->xe, ext.pad) || 1383 XE_IOCTL_DBG(oa->xe, ext.name >= ARRAY_SIZE(xe_oa_user_extension_funcs))) 1384 return -EINVAL; 1385 1386 idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_oa_user_extension_funcs)); 1387 err = xe_oa_user_extension_funcs[idx](oa, from, extension, param); 1388 if (XE_IOCTL_DBG(oa->xe, err)) 1389 return err; 1390 1391 if (ext.next_extension) 1392 return xe_oa_user_extensions(oa, from, ext.next_extension, ++ext_number, param); 1393 1394 return 0; 1395} 1396 1397static int xe_oa_parse_syncs(struct xe_oa *oa, 1398 struct xe_oa_stream *stream, 1399 struct xe_oa_open_param *param) 1400{ 1401 int ret, num_syncs, num_ufence = 0; 1402 1403 if (param->num_syncs && !param->syncs_user) { 1404 drm_dbg(&oa->xe->drm, "num_syncs specified without sync array\n"); 1405 ret = -EINVAL; 1406 goto exit; 1407 } 1408 1409 if (param->num_syncs) { 1410 param->syncs = kcalloc(param->num_syncs, sizeof(*param->syncs), GFP_KERNEL); 1411 if (!param->syncs) { 1412 ret = -ENOMEM; 1413 goto exit; 1414 } 1415 } 1416 1417 for (num_syncs = 0; num_syncs < param->num_syncs; num_syncs++) { 1418 ret = xe_sync_entry_parse(oa->xe, param->xef, ¶m->syncs[num_syncs], 1419 ¶m->syncs_user[num_syncs], 1420 stream->ufence_syncobj, 1421 ++stream->ufence_timeline_value, 0); 1422 if (ret) 1423 goto err_syncs; 1424 1425 if (xe_sync_is_ufence(¶m->syncs[num_syncs])) 1426 num_ufence++; 1427 } 1428 1429 if (XE_IOCTL_DBG(oa->xe, num_ufence > 1)) { 1430 ret = -EINVAL; 1431 goto err_syncs; 1432 } 1433 1434 return 0; 1435 1436err_syncs: 1437 while (num_syncs--) 1438 xe_sync_entry_cleanup(¶m->syncs[num_syncs]); 1439 kfree(param->syncs); 1440exit: 1441 return ret; 1442} 1443 1444static void xe_oa_stream_enable(struct xe_oa_stream *stream) 1445{ 1446 stream->pollin = false; 1447 1448 xe_oa_enable(stream); 1449 1450 if (stream->sample) 1451 hrtimer_start(&stream->poll_check_timer, 1452 ns_to_ktime(stream->poll_period_ns), 1453 HRTIMER_MODE_REL_PINNED); 1454} 1455 1456static void xe_oa_stream_disable(struct xe_oa_stream *stream) 1457{ 1458 xe_oa_disable(stream); 1459 1460 if (stream->sample) 1461 hrtimer_cancel(&stream->poll_check_timer); 1462 1463 /* Update stream->oa_buffer.tail to allow any final reports to be read */ 1464 if (xe_oa_buffer_check_unlocked(stream)) 1465 wake_up(&stream->poll_wq); 1466} 1467 1468static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream) 1469{ 1470 struct xe_exec_queue *q = stream->exec_q; 1471 int ret1, ret2; 1472 1473 /* Best effort recovery: try to revert both to original, irrespective of error */ 1474 ret1 = q->ops->set_timeslice(q, stream->hwe->eclass->sched_props.timeslice_us); 1475 ret2 = q->ops->set_preempt_timeout(q, stream->hwe->eclass->sched_props.preempt_timeout_us); 1476 if (ret1 || ret2) 1477 goto err; 1478 return 0; 1479err: 1480 drm_dbg(&stream->oa->xe->drm, "%s failed ret1 %d ret2 %d\n", __func__, ret1, ret2); 1481 return ret1 ?: ret2; 1482} 1483 1484static int xe_oa_disable_preempt_timeslice(struct xe_oa_stream *stream) 1485{ 1486 struct xe_exec_queue *q = stream->exec_q; 1487 int ret; 1488 1489 /* Setting values to 0 will disable timeslice and preempt_timeout */ 1490 ret = q->ops->set_timeslice(q, 0); 1491 if (ret) 1492 goto err; 1493 1494 ret = q->ops->set_preempt_timeout(q, 0); 1495 if (ret) 1496 goto err; 1497 1498 return 0; 1499err: 1500 xe_oa_enable_preempt_timeslice(stream); 1501 drm_dbg(&stream->oa->xe->drm, "%s failed %d\n", __func__, ret); 1502 return ret; 1503} 1504 1505static int xe_oa_enable_locked(struct xe_oa_stream *stream) 1506{ 1507 if (stream->enabled) 1508 return 0; 1509 1510 if (stream->no_preempt) { 1511 int ret = xe_oa_disable_preempt_timeslice(stream); 1512 1513 if (ret) 1514 return ret; 1515 } 1516 1517 xe_oa_stream_enable(stream); 1518 1519 stream->enabled = true; 1520 return 0; 1521} 1522 1523static int xe_oa_disable_locked(struct xe_oa_stream *stream) 1524{ 1525 int ret = 0; 1526 1527 if (!stream->enabled) 1528 return 0; 1529 1530 xe_oa_stream_disable(stream); 1531 1532 if (stream->no_preempt) 1533 ret = xe_oa_enable_preempt_timeslice(stream); 1534 1535 stream->enabled = false; 1536 return ret; 1537} 1538 1539static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg) 1540{ 1541 struct xe_oa_open_param param = {}; 1542 long ret = stream->oa_config->id; 1543 struct xe_oa_config *config; 1544 int err; 1545 1546 err = xe_oa_user_extensions(stream->oa, XE_OA_USER_EXTN_FROM_CONFIG, arg, 0, ¶m); 1547 if (err) 1548 return err; 1549 1550 config = xe_oa_get_oa_config(stream->oa, param.metric_set); 1551 if (!config) 1552 return -ENODEV; 1553 1554 param.xef = stream->xef; 1555 err = xe_oa_parse_syncs(stream->oa, stream, ¶m); 1556 if (err) 1557 goto err_config_put; 1558 1559 stream->num_syncs = param.num_syncs; 1560 stream->syncs = param.syncs; 1561 1562 err = xe_oa_emit_oa_config(stream, config); 1563 if (!err) { 1564 config = xchg(&stream->oa_config, config); 1565 drm_dbg(&stream->oa->xe->drm, "changed to oa config uuid=%s\n", 1566 stream->oa_config->uuid); 1567 } 1568 1569err_config_put: 1570 xe_oa_config_put(config); 1571 1572 return err ?: ret; 1573} 1574 1575static long xe_oa_status_locked(struct xe_oa_stream *stream, unsigned long arg) 1576{ 1577 struct drm_xe_oa_stream_status status = {}; 1578 void __user *uaddr = (void __user *)arg; 1579 1580 /* Map from register to uapi bits */ 1581 if (stream->oa_status & OASTATUS_REPORT_LOST) 1582 status.oa_status |= DRM_XE_OASTATUS_REPORT_LOST; 1583 if (stream->oa_status & OASTATUS_BUFFER_OVERFLOW) 1584 status.oa_status |= DRM_XE_OASTATUS_BUFFER_OVERFLOW; 1585 if (stream->oa_status & OASTATUS_COUNTER_OVERFLOW) 1586 status.oa_status |= DRM_XE_OASTATUS_COUNTER_OVERFLOW; 1587 if (stream->oa_status & OASTATUS_MMIO_TRG_Q_FULL) 1588 status.oa_status |= DRM_XE_OASTATUS_MMIO_TRG_Q_FULL; 1589 1590 if (copy_to_user(uaddr, &status, sizeof(status))) 1591 return -EFAULT; 1592 1593 return 0; 1594} 1595 1596static long xe_oa_info_locked(struct xe_oa_stream *stream, unsigned long arg) 1597{ 1598 struct drm_xe_oa_stream_info info = { .oa_buf_size = xe_bo_size(stream->oa_buffer.bo), }; 1599 void __user *uaddr = (void __user *)arg; 1600 1601 if (copy_to_user(uaddr, &info, sizeof(info))) 1602 return -EFAULT; 1603 1604 return 0; 1605} 1606 1607static long xe_oa_ioctl_locked(struct xe_oa_stream *stream, 1608 unsigned int cmd, 1609 unsigned long arg) 1610{ 1611 switch (cmd) { 1612 case DRM_XE_OBSERVATION_IOCTL_ENABLE: 1613 return xe_oa_enable_locked(stream); 1614 case DRM_XE_OBSERVATION_IOCTL_DISABLE: 1615 return xe_oa_disable_locked(stream); 1616 case DRM_XE_OBSERVATION_IOCTL_CONFIG: 1617 return xe_oa_config_locked(stream, arg); 1618 case DRM_XE_OBSERVATION_IOCTL_STATUS: 1619 return xe_oa_status_locked(stream, arg); 1620 case DRM_XE_OBSERVATION_IOCTL_INFO: 1621 return xe_oa_info_locked(stream, arg); 1622 } 1623 1624 return -EINVAL; 1625} 1626 1627static long xe_oa_ioctl(struct file *file, 1628 unsigned int cmd, 1629 unsigned long arg) 1630{ 1631 struct xe_oa_stream *stream = file->private_data; 1632 long ret; 1633 1634 mutex_lock(&stream->stream_lock); 1635 ret = xe_oa_ioctl_locked(stream, cmd, arg); 1636 mutex_unlock(&stream->stream_lock); 1637 1638 return ret; 1639} 1640 1641static void xe_oa_destroy_locked(struct xe_oa_stream *stream) 1642{ 1643 if (stream->enabled) 1644 xe_oa_disable_locked(stream); 1645 1646 xe_oa_stream_destroy(stream); 1647 1648 if (stream->exec_q) 1649 xe_exec_queue_put(stream->exec_q); 1650 1651 drm_syncobj_put(stream->ufence_syncobj); 1652 kfree(stream); 1653} 1654 1655static int xe_oa_release(struct inode *inode, struct file *file) 1656{ 1657 struct xe_oa_stream *stream = file->private_data; 1658 struct xe_gt *gt = stream->gt; 1659 1660 xe_pm_runtime_get(gt_to_xe(gt)); 1661 mutex_lock(>->oa.gt_lock); 1662 xe_oa_destroy_locked(stream); 1663 mutex_unlock(>->oa.gt_lock); 1664 xe_pm_runtime_put(gt_to_xe(gt)); 1665 1666 /* Release the reference the OA stream kept on the driver */ 1667 drm_dev_put(>_to_xe(gt)->drm); 1668 1669 return 0; 1670} 1671 1672static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) 1673{ 1674 struct xe_oa_stream *stream = file->private_data; 1675 struct xe_bo *bo = stream->oa_buffer.bo; 1676 unsigned long start = vma->vm_start; 1677 int i, ret; 1678 1679 if (xe_observation_paranoid && !perfmon_capable()) { 1680 drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n"); 1681 return -EACCES; 1682 } 1683 1684 /* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */ 1685 if (vma->vm_end - vma->vm_start != xe_bo_size(stream->oa_buffer.bo)) { 1686 drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n"); 1687 return -EINVAL; 1688 } 1689 1690 /* 1691 * Only support VM_READ, enforce MAP_PRIVATE by checking for 1692 * VM_MAYSHARE, don't copy the vma on fork 1693 */ 1694 if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_SHARED | VM_MAYSHARE)) { 1695 drm_dbg(&stream->oa->xe->drm, "mmap must be read only\n"); 1696 return -EINVAL; 1697 } 1698 vm_flags_mod(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY, 1699 VM_MAYWRITE | VM_MAYEXEC); 1700 1701 xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == vma_pages(vma)); 1702 for (i = 0; i < bo->ttm.ttm->num_pages; i++) { 1703 ret = remap_pfn_range(vma, start, page_to_pfn(bo->ttm.ttm->pages[i]), 1704 PAGE_SIZE, vma->vm_page_prot); 1705 if (ret) 1706 break; 1707 1708 start += PAGE_SIZE; 1709 } 1710 1711 return ret; 1712} 1713 1714static const struct file_operations xe_oa_fops = { 1715 .owner = THIS_MODULE, 1716 .release = xe_oa_release, 1717 .poll = xe_oa_poll, 1718 .read = xe_oa_read, 1719 .unlocked_ioctl = xe_oa_ioctl, 1720 .mmap = xe_oa_mmap, 1721}; 1722 1723static int xe_oa_stream_init(struct xe_oa_stream *stream, 1724 struct xe_oa_open_param *param) 1725{ 1726 struct xe_gt *gt = param->hwe->gt; 1727 int ret; 1728 1729 stream->exec_q = param->exec_q; 1730 stream->poll_period_ns = DEFAULT_POLL_PERIOD_NS; 1731 stream->oa_unit = param->oa_unit; 1732 stream->hwe = param->hwe; 1733 stream->gt = stream->hwe->gt; 1734 stream->oa_buffer.format = &stream->oa->oa_formats[param->oa_format]; 1735 1736 stream->sample = param->sample; 1737 stream->periodic = param->period_exponent >= 0; 1738 stream->period_exponent = param->period_exponent; 1739 stream->no_preempt = param->no_preempt; 1740 stream->wait_num_reports = param->wait_num_reports; 1741 1742 stream->xef = xe_file_get(param->xef); 1743 stream->num_syncs = param->num_syncs; 1744 stream->syncs = param->syncs; 1745 1746 /* 1747 * For Xe2+, when overrun mode is enabled, there are no partial reports at the end 1748 * of buffer, making the OA buffer effectively a non-power-of-2 size circular 1749 * buffer whose size, circ_size, is a multiple of the report size 1750 */ 1751 if (GRAPHICS_VER(stream->oa->xe) >= 20 && 1752 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG && stream->sample) 1753 stream->oa_buffer.circ_size = 1754 param->oa_buffer_size - 1755 param->oa_buffer_size % stream->oa_buffer.format->size; 1756 else 1757 stream->oa_buffer.circ_size = param->oa_buffer_size; 1758 1759 stream->oa_config = xe_oa_get_oa_config(stream->oa, param->metric_set); 1760 if (!stream->oa_config) { 1761 drm_dbg(&stream->oa->xe->drm, "Invalid OA config id=%i\n", param->metric_set); 1762 ret = -EINVAL; 1763 goto exit; 1764 } 1765 1766 /* 1767 * GuC reset of engines causes OA to lose configuration 1768 * state. Prevent this by overriding GUCRC mode. 1769 */ 1770 if (XE_GT_WA(stream->gt, 1509372804)) { 1771 ret = xe_guc_pc_override_gucrc_mode(>->uc.guc.pc, 1772 SLPC_GUCRC_MODE_GUCRC_NO_RC6); 1773 if (ret) 1774 goto err_free_configs; 1775 1776 stream->override_gucrc = true; 1777 } 1778 1779 /* Take runtime pm ref and forcewake to disable RC6 */ 1780 xe_pm_runtime_get(stream->oa->xe); 1781 stream->fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 1782 if (!xe_force_wake_ref_has_domain(stream->fw_ref, XE_FORCEWAKE_ALL)) { 1783 ret = -ETIMEDOUT; 1784 goto err_fw_put; 1785 } 1786 1787 ret = xe_oa_alloc_oa_buffer(stream, param->oa_buffer_size); 1788 if (ret) 1789 goto err_fw_put; 1790 1791 stream->k_exec_q = xe_exec_queue_create(stream->oa->xe, NULL, 1792 BIT(stream->hwe->logical_instance), 1, 1793 stream->hwe, EXEC_QUEUE_FLAG_KERNEL, 0); 1794 if (IS_ERR(stream->k_exec_q)) { 1795 ret = PTR_ERR(stream->k_exec_q); 1796 drm_err(&stream->oa->xe->drm, "gt%d, hwe %s, xe_exec_queue_create failed=%d", 1797 stream->gt->info.id, stream->hwe->name, ret); 1798 goto err_free_oa_buf; 1799 } 1800 1801 ret = xe_oa_enable_metric_set(stream); 1802 if (ret) { 1803 drm_dbg(&stream->oa->xe->drm, "Unable to enable metric set\n"); 1804 goto err_put_k_exec_q; 1805 } 1806 1807 drm_dbg(&stream->oa->xe->drm, "opening stream oa config uuid=%s\n", 1808 stream->oa_config->uuid); 1809 1810 WRITE_ONCE(stream->oa_unit->exclusive_stream, stream); 1811 1812 hrtimer_setup(&stream->poll_check_timer, xe_oa_poll_check_timer_cb, CLOCK_MONOTONIC, 1813 HRTIMER_MODE_REL); 1814 init_waitqueue_head(&stream->poll_wq); 1815 1816 spin_lock_init(&stream->oa_buffer.ptr_lock); 1817 mutex_init(&stream->stream_lock); 1818 1819 return 0; 1820 1821err_put_k_exec_q: 1822 xe_oa_disable_metric_set(stream); 1823 xe_exec_queue_put(stream->k_exec_q); 1824err_free_oa_buf: 1825 xe_oa_free_oa_buffer(stream); 1826err_fw_put: 1827 xe_force_wake_put(gt_to_fw(gt), stream->fw_ref); 1828 xe_pm_runtime_put(stream->oa->xe); 1829 if (stream->override_gucrc) 1830 xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(>->uc.guc.pc)); 1831err_free_configs: 1832 xe_oa_free_configs(stream); 1833exit: 1834 xe_file_put(stream->xef); 1835 return ret; 1836} 1837 1838static int xe_oa_stream_open_ioctl_locked(struct xe_oa *oa, 1839 struct xe_oa_open_param *param) 1840{ 1841 struct xe_oa_stream *stream; 1842 struct drm_syncobj *ufence_syncobj; 1843 int stream_fd; 1844 int ret; 1845 1846 /* We currently only allow exclusive access */ 1847 if (param->oa_unit->exclusive_stream) { 1848 drm_dbg(&oa->xe->drm, "OA unit already in use\n"); 1849 ret = -EBUSY; 1850 goto exit; 1851 } 1852 1853 ret = drm_syncobj_create(&ufence_syncobj, DRM_SYNCOBJ_CREATE_SIGNALED, 1854 NULL); 1855 if (ret) 1856 goto exit; 1857 1858 stream = kzalloc(sizeof(*stream), GFP_KERNEL); 1859 if (!stream) { 1860 ret = -ENOMEM; 1861 goto err_syncobj; 1862 } 1863 stream->ufence_syncobj = ufence_syncobj; 1864 stream->oa = oa; 1865 1866 ret = xe_oa_parse_syncs(oa, stream, param); 1867 if (ret) 1868 goto err_free; 1869 1870 ret = xe_oa_stream_init(stream, param); 1871 if (ret) { 1872 while (param->num_syncs--) 1873 xe_sync_entry_cleanup(¶m->syncs[param->num_syncs]); 1874 kfree(param->syncs); 1875 goto err_free; 1876 } 1877 1878 if (!param->disabled) { 1879 ret = xe_oa_enable_locked(stream); 1880 if (ret) 1881 goto err_destroy; 1882 } 1883 1884 stream_fd = anon_inode_getfd("[xe_oa]", &xe_oa_fops, stream, 0); 1885 if (stream_fd < 0) { 1886 ret = stream_fd; 1887 goto err_disable; 1888 } 1889 1890 /* Hold a reference on the drm device till stream_fd is released */ 1891 drm_dev_get(&stream->oa->xe->drm); 1892 1893 return stream_fd; 1894err_disable: 1895 if (!param->disabled) 1896 xe_oa_disable_locked(stream); 1897err_destroy: 1898 xe_oa_stream_destroy(stream); 1899err_free: 1900 kfree(stream); 1901err_syncobj: 1902 drm_syncobj_put(ufence_syncobj); 1903exit: 1904 return ret; 1905} 1906 1907/** 1908 * xe_oa_timestamp_frequency - Return OA timestamp frequency 1909 * @gt: @xe_gt 1910 * 1911 * OA timestamp frequency = CS timestamp frequency in most platforms. On some 1912 * platforms OA unit ignores the CTC_SHIFT and the 2 timestamps differ. In such 1913 * cases, return the adjusted CS timestamp frequency to the user. 1914 */ 1915u32 xe_oa_timestamp_frequency(struct xe_gt *gt) 1916{ 1917 u32 reg, shift; 1918 1919 if (XE_GT_WA(gt, 18013179988) || XE_GT_WA(gt, 14015568240)) { 1920 xe_pm_runtime_get(gt_to_xe(gt)); 1921 reg = xe_mmio_read32(>->mmio, RPM_CONFIG0); 1922 xe_pm_runtime_put(gt_to_xe(gt)); 1923 1924 shift = REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, reg); 1925 return gt->info.reference_clock << (3 - shift); 1926 } else { 1927 return gt->info.reference_clock; 1928 } 1929} 1930 1931static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent) 1932{ 1933 u64 nom = (2ULL << exponent) * NSEC_PER_SEC; 1934 u32 den = xe_oa_timestamp_frequency(gt); 1935 1936 return div_u64(nom + den - 1, den); 1937} 1938 1939static bool oa_unit_supports_oa_format(struct xe_oa_open_param *param, int type) 1940{ 1941 switch (param->oa_unit->type) { 1942 case DRM_XE_OA_UNIT_TYPE_OAG: 1943 return type == DRM_XE_OA_FMT_TYPE_OAG || type == DRM_XE_OA_FMT_TYPE_OAR || 1944 type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC; 1945 case DRM_XE_OA_UNIT_TYPE_OAM: 1946 case DRM_XE_OA_UNIT_TYPE_OAM_SAG: 1947 return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; 1948 default: 1949 return false; 1950 } 1951} 1952 1953/** 1954 * xe_oa_unit_id - Return OA unit ID for a hardware engine 1955 * @hwe: @xe_hw_engine 1956 * 1957 * Return OA unit ID for a hardware engine when available 1958 */ 1959u16 xe_oa_unit_id(struct xe_hw_engine *hwe) 1960{ 1961 return hwe->oa_unit && hwe->oa_unit->num_engines ? 1962 hwe->oa_unit->oa_unit_id : U16_MAX; 1963} 1964 1965/* A hwe must be assigned to stream/oa_unit for batch submissions */ 1966static int xe_oa_assign_hwe(struct xe_oa *oa, struct xe_oa_open_param *param) 1967{ 1968 struct xe_hw_engine *hwe; 1969 enum xe_hw_engine_id id; 1970 int ret = 0; 1971 1972 /* If not provided, OA unit defaults to OA unit 0 as per uapi */ 1973 if (!param->oa_unit) 1974 param->oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0]; 1975 1976 /* When we have an exec_q, get hwe from the exec_q */ 1977 if (param->exec_q) { 1978 param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class, 1979 param->engine_instance, true); 1980 if (!param->hwe || param->hwe->oa_unit != param->oa_unit) 1981 goto err; 1982 goto out; 1983 } 1984 1985 /* Else just get the first hwe attached to the oa unit */ 1986 for_each_hw_engine(hwe, param->oa_unit->gt, id) { 1987 if (hwe->oa_unit == param->oa_unit) { 1988 param->hwe = hwe; 1989 goto out; 1990 } 1991 } 1992 1993 /* If we still didn't find a hwe, just get one with a valid oa_unit from the same gt */ 1994 for_each_hw_engine(hwe, param->oa_unit->gt, id) { 1995 if (!hwe->oa_unit) 1996 continue; 1997 1998 param->hwe = hwe; 1999 goto out; 2000 } 2001err: 2002 drm_dbg(&oa->xe->drm, "Unable to find hwe (%d, %d) for OA unit ID %d\n", 2003 param->exec_q ? param->exec_q->class : -1, 2004 param->engine_instance, param->oa_unit->oa_unit_id); 2005 ret = -EINVAL; 2006out: 2007 return ret; 2008} 2009 2010/** 2011 * xe_oa_stream_open_ioctl - Opens an OA stream 2012 * @dev: @drm_device 2013 * @data: pointer to struct @drm_xe_oa_config 2014 * @file: @drm_file 2015 * 2016 * The functions opens an OA stream. An OA stream, opened with specified 2017 * properties, enables OA counter samples to be collected, either 2018 * periodically (time based sampling), or on request (using OA queries) 2019 */ 2020int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *file) 2021{ 2022 struct xe_device *xe = to_xe_device(dev); 2023 struct xe_oa *oa = &xe->oa; 2024 struct xe_file *xef = to_xe_file(file); 2025 struct xe_oa_open_param param = {}; 2026 const struct xe_oa_format *f; 2027 bool privileged_op = true; 2028 int ret; 2029 2030 if (!oa->xe) { 2031 drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); 2032 return -ENODEV; 2033 } 2034 2035 param.xef = xef; 2036 param.period_exponent = -1; 2037 ret = xe_oa_user_extensions(oa, XE_OA_USER_EXTN_FROM_OPEN, data, 0, ¶m); 2038 if (ret) 2039 return ret; 2040 2041 if (param.exec_queue_id > 0) { 2042 param.exec_q = xe_exec_queue_lookup(xef, param.exec_queue_id); 2043 if (XE_IOCTL_DBG(oa->xe, !param.exec_q)) 2044 return -ENOENT; 2045 2046 if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1)) 2047 return -EOPNOTSUPP; 2048 } 2049 2050 /* 2051 * Query based sampling (using MI_REPORT_PERF_COUNT) with OAR/OAC, 2052 * without global stream access, can be an unprivileged operation 2053 */ 2054 if (param.exec_q && !param.sample) 2055 privileged_op = false; 2056 2057 if (param.no_preempt) { 2058 if (!param.exec_q) { 2059 drm_dbg(&oa->xe->drm, "Preemption disable without exec_q!\n"); 2060 ret = -EINVAL; 2061 goto err_exec_q; 2062 } 2063 privileged_op = true; 2064 } 2065 2066 if (privileged_op && xe_observation_paranoid && !perfmon_capable()) { 2067 drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n"); 2068 ret = -EACCES; 2069 goto err_exec_q; 2070 } 2071 2072 if (!param.exec_q && !param.sample) { 2073 drm_dbg(&oa->xe->drm, "Only OA report sampling supported\n"); 2074 ret = -EINVAL; 2075 goto err_exec_q; 2076 } 2077 2078 ret = xe_oa_assign_hwe(oa, ¶m); 2079 if (ret) 2080 goto err_exec_q; 2081 2082 f = &oa->oa_formats[param.oa_format]; 2083 if (!param.oa_format || !f->size || 2084 !oa_unit_supports_oa_format(¶m, f->type)) { 2085 drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n", 2086 param.oa_format, f->type, f->size, param.hwe->class); 2087 ret = -EINVAL; 2088 goto err_exec_q; 2089 } 2090 2091 if (param.period_exponent >= 0) { 2092 u64 oa_period, oa_freq_hz; 2093 2094 /* Requesting samples from OAG buffer is a privileged operation */ 2095 if (!param.sample) { 2096 drm_dbg(&oa->xe->drm, "OA_EXPONENT specified without SAMPLE_OA\n"); 2097 ret = -EINVAL; 2098 goto err_exec_q; 2099 } 2100 oa_period = oa_exponent_to_ns(param.hwe->gt, param.period_exponent); 2101 oa_freq_hz = div64_u64(NSEC_PER_SEC, oa_period); 2102 drm_dbg(&oa->xe->drm, "Using periodic sampling freq %lld Hz\n", oa_freq_hz); 2103 } 2104 2105 if (!param.oa_buffer_size) 2106 param.oa_buffer_size = DEFAULT_XE_OA_BUFFER_SIZE; 2107 2108 if (!param.wait_num_reports) 2109 param.wait_num_reports = 1; 2110 if (param.wait_num_reports > param.oa_buffer_size / f->size) { 2111 drm_dbg(&oa->xe->drm, "wait_num_reports %d\n", param.wait_num_reports); 2112 ret = -EINVAL; 2113 goto err_exec_q; 2114 } 2115 2116 mutex_lock(¶m.hwe->gt->oa.gt_lock); 2117 ret = xe_oa_stream_open_ioctl_locked(oa, ¶m); 2118 mutex_unlock(¶m.hwe->gt->oa.gt_lock); 2119 if (ret < 0) 2120 goto err_exec_q; 2121 2122 return ret; 2123 2124err_exec_q: 2125 if (param.exec_q) 2126 xe_exec_queue_put(param.exec_q); 2127 return ret; 2128} 2129 2130static bool xe_oa_is_valid_flex_addr(struct xe_oa *oa, u32 addr) 2131{ 2132 static const struct xe_reg flex_eu_regs[] = { 2133 EU_PERF_CNTL0, 2134 EU_PERF_CNTL1, 2135 EU_PERF_CNTL2, 2136 EU_PERF_CNTL3, 2137 EU_PERF_CNTL4, 2138 EU_PERF_CNTL5, 2139 EU_PERF_CNTL6, 2140 }; 2141 int i; 2142 2143 for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) { 2144 if (flex_eu_regs[i].addr == addr) 2145 return true; 2146 } 2147 return false; 2148} 2149 2150static bool xe_oa_reg_in_range_table(u32 addr, const struct xe_mmio_range *table) 2151{ 2152 while (table->start && table->end) { 2153 if (addr >= table->start && addr <= table->end) 2154 return true; 2155 2156 table++; 2157 } 2158 2159 return false; 2160} 2161 2162static const struct xe_mmio_range xehp_oa_b_counters[] = { 2163 { .start = 0xdc48, .end = 0xdc48 }, /* OAA_ENABLE_REG */ 2164 { .start = 0xdd00, .end = 0xdd48 }, /* OAG_LCE0_0 - OAA_LENABLE_REG */ 2165 {} 2166}; 2167 2168static const struct xe_mmio_range gen12_oa_b_counters[] = { 2169 { .start = 0x2b2c, .end = 0x2b2c }, /* OAG_OA_PESS */ 2170 { .start = 0xd900, .end = 0xd91c }, /* OAG_OASTARTTRIG[1-8] */ 2171 { .start = 0xd920, .end = 0xd93c }, /* OAG_OAREPORTTRIG1[1-8] */ 2172 { .start = 0xd940, .end = 0xd97c }, /* OAG_CEC[0-7][0-1] */ 2173 { .start = 0xdc00, .end = 0xdc3c }, /* OAG_SCEC[0-7][0-1] */ 2174 { .start = 0xdc40, .end = 0xdc40 }, /* OAG_SPCTR_CNF */ 2175 { .start = 0xdc44, .end = 0xdc44 }, /* OAA_DBG_REG */ 2176 {} 2177}; 2178 2179static const struct xe_mmio_range mtl_oam_b_counters[] = { 2180 { .start = 0x393000, .end = 0x39301c }, /* OAM_STARTTRIG1[1-8] */ 2181 { .start = 0x393020, .end = 0x39303c }, /* OAM_REPORTTRIG1[1-8] */ 2182 { .start = 0x393040, .end = 0x39307c }, /* OAM_CEC[0-7][0-1] */ 2183 { .start = 0x393200, .end = 0x39323C }, /* MPES[0-7] */ 2184 {} 2185}; 2186 2187static const struct xe_mmio_range xe2_oa_b_counters[] = { 2188 { .start = 0x393200, .end = 0x39323C }, /* MPES_0_MPES_SAG - MPES_7_UPPER_MPES_SAG */ 2189 { .start = 0x394200, .end = 0x39423C }, /* MPES_0_MPES_SCMI0 - MPES_7_UPPER_MPES_SCMI0 */ 2190 { .start = 0x394A00, .end = 0x394A3C }, /* MPES_0_MPES_SCMI1 - MPES_7_UPPER_MPES_SCMI1 */ 2191 {}, 2192}; 2193 2194static bool xe_oa_is_valid_b_counter_addr(struct xe_oa *oa, u32 addr) 2195{ 2196 return xe_oa_reg_in_range_table(addr, xehp_oa_b_counters) || 2197 xe_oa_reg_in_range_table(addr, gen12_oa_b_counters) || 2198 xe_oa_reg_in_range_table(addr, mtl_oam_b_counters) || 2199 (GRAPHICS_VER(oa->xe) >= 20 && 2200 xe_oa_reg_in_range_table(addr, xe2_oa_b_counters)); 2201} 2202 2203static const struct xe_mmio_range mtl_oa_mux_regs[] = { 2204 { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */ 2205 { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */ 2206 { .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */ 2207 { .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */ 2208 { .start = 0x38d100, .end = 0x38d114}, /* VISACTL */ 2209 {} 2210}; 2211 2212static const struct xe_mmio_range gen12_oa_mux_regs[] = { 2213 { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */ 2214 { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */ 2215 { .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */ 2216 { .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */ 2217 { .start = 0x20cc, .end = 0x20cc }, /* WAIT_FOR_RC6_EXIT */ 2218 {} 2219}; 2220 2221static const struct xe_mmio_range xe2_oa_mux_regs[] = { 2222 { .start = 0x5194, .end = 0x5194 }, /* SYS_MEM_LAT_MEASURE_MERTF_GRP_3D */ 2223 { .start = 0x8704, .end = 0x8704 }, /* LMEM_LAT_MEASURE_MCFG_GRP */ 2224 { .start = 0xB01C, .end = 0xB01C }, /* LNCF_MISC_CONFIG_REGISTER0 */ 2225 { .start = 0xB1BC, .end = 0xB1BC }, /* L3_BANK_LAT_MEASURE_LBCF_GFX */ 2226 { .start = 0xD0E0, .end = 0xD0F4 }, /* VISACTL */ 2227 { .start = 0xE18C, .end = 0xE18C }, /* SAMPLER_MODE */ 2228 { .start = 0xE590, .end = 0xE590 }, /* TDL_LSC_LAT_MEASURE_TDL_GFX */ 2229 { .start = 0x13000, .end = 0x137FC }, /* PES_0_PESL0 - PES_63_UPPER_PESL3 */ 2230 {}, 2231}; 2232 2233static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr) 2234{ 2235 if (GRAPHICS_VER(oa->xe) >= 20) 2236 return xe_oa_reg_in_range_table(addr, xe2_oa_mux_regs); 2237 else if (GRAPHICS_VERx100(oa->xe) >= 1270) 2238 return xe_oa_reg_in_range_table(addr, mtl_oa_mux_regs); 2239 else 2240 return xe_oa_reg_in_range_table(addr, gen12_oa_mux_regs); 2241} 2242 2243static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr) 2244{ 2245 return xe_oa_is_valid_flex_addr(oa, addr) || 2246 xe_oa_is_valid_b_counter_addr(oa, addr) || 2247 xe_oa_is_valid_mux_addr(oa, addr); 2248} 2249 2250static struct xe_oa_reg * 2251xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr), 2252 u32 __user *regs, u32 n_regs) 2253{ 2254 struct xe_oa_reg *oa_regs; 2255 int err; 2256 u32 i; 2257 2258 oa_regs = kmalloc_array(n_regs, sizeof(*oa_regs), GFP_KERNEL); 2259 if (!oa_regs) 2260 return ERR_PTR(-ENOMEM); 2261 2262 for (i = 0; i < n_regs; i++) { 2263 u32 addr, value; 2264 2265 err = get_user(addr, regs); 2266 if (err) 2267 goto addr_err; 2268 2269 if (!is_valid(oa, addr)) { 2270 drm_dbg(&oa->xe->drm, "Invalid oa_reg address: %X\n", addr); 2271 err = -EINVAL; 2272 goto addr_err; 2273 } 2274 2275 err = get_user(value, regs + 1); 2276 if (err) 2277 goto addr_err; 2278 2279 oa_regs[i].addr = XE_REG(addr); 2280 oa_regs[i].value = value; 2281 2282 regs += 2; 2283 } 2284 2285 return oa_regs; 2286 2287addr_err: 2288 kfree(oa_regs); 2289 return ERR_PTR(err); 2290} 2291ALLOW_ERROR_INJECTION(xe_oa_alloc_regs, ERRNO); 2292 2293static ssize_t show_dynamic_id(struct kobject *kobj, 2294 struct kobj_attribute *attr, 2295 char *buf) 2296{ 2297 struct xe_oa_config *oa_config = 2298 container_of(attr, typeof(*oa_config), sysfs_metric_id); 2299 2300 return sysfs_emit(buf, "%d\n", oa_config->id); 2301} 2302 2303static int create_dynamic_oa_sysfs_entry(struct xe_oa *oa, 2304 struct xe_oa_config *oa_config) 2305{ 2306 sysfs_attr_init(&oa_config->sysfs_metric_id.attr); 2307 oa_config->sysfs_metric_id.attr.name = "id"; 2308 oa_config->sysfs_metric_id.attr.mode = 0444; 2309 oa_config->sysfs_metric_id.show = show_dynamic_id; 2310 oa_config->sysfs_metric_id.store = NULL; 2311 2312 oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr; 2313 oa_config->attrs[1] = NULL; 2314 2315 oa_config->sysfs_metric.name = oa_config->uuid; 2316 oa_config->sysfs_metric.attrs = oa_config->attrs; 2317 2318 return sysfs_create_group(oa->metrics_kobj, &oa_config->sysfs_metric); 2319} 2320 2321/** 2322 * xe_oa_add_config_ioctl - Adds one OA config 2323 * @dev: @drm_device 2324 * @data: pointer to struct @drm_xe_oa_config 2325 * @file: @drm_file 2326 * 2327 * The functions adds an OA config to the set of OA configs maintained in 2328 * the kernel. The config determines which OA metrics are collected for an 2329 * OA stream. 2330 */ 2331int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file) 2332{ 2333 struct xe_device *xe = to_xe_device(dev); 2334 struct xe_oa *oa = &xe->oa; 2335 struct drm_xe_oa_config param; 2336 struct drm_xe_oa_config *arg = ¶m; 2337 struct xe_oa_config *oa_config, *tmp; 2338 struct xe_oa_reg *regs; 2339 int err, id; 2340 2341 if (!oa->xe) { 2342 drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); 2343 return -ENODEV; 2344 } 2345 2346 if (xe_observation_paranoid && !perfmon_capable()) { 2347 drm_dbg(&oa->xe->drm, "Insufficient privileges to add xe OA config\n"); 2348 return -EACCES; 2349 } 2350 2351 err = copy_from_user(¶m, u64_to_user_ptr(data), sizeof(param)); 2352 if (XE_IOCTL_DBG(oa->xe, err)) 2353 return -EFAULT; 2354 2355 if (XE_IOCTL_DBG(oa->xe, arg->extensions) || 2356 XE_IOCTL_DBG(oa->xe, !arg->regs_ptr) || 2357 XE_IOCTL_DBG(oa->xe, !arg->n_regs)) 2358 return -EINVAL; 2359 2360 oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL); 2361 if (!oa_config) 2362 return -ENOMEM; 2363 2364 oa_config->oa = oa; 2365 kref_init(&oa_config->ref); 2366 2367 if (!uuid_is_valid(arg->uuid)) { 2368 drm_dbg(&oa->xe->drm, "Invalid uuid format for OA config\n"); 2369 err = -EINVAL; 2370 goto reg_err; 2371 } 2372 2373 /* Last character in oa_config->uuid will be 0 because oa_config is kzalloc */ 2374 memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid)); 2375 2376 oa_config->regs_len = arg->n_regs; 2377 regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg_addr, 2378 u64_to_user_ptr(arg->regs_ptr), 2379 arg->n_regs); 2380 if (IS_ERR(regs)) { 2381 drm_dbg(&oa->xe->drm, "Failed to create OA config for mux_regs\n"); 2382 err = PTR_ERR(regs); 2383 goto reg_err; 2384 } 2385 oa_config->regs = regs; 2386 2387 err = mutex_lock_interruptible(&oa->metrics_lock); 2388 if (err) 2389 goto reg_err; 2390 2391 /* We shouldn't have too many configs, so this iteration shouldn't be too costly */ 2392 idr_for_each_entry(&oa->metrics_idr, tmp, id) { 2393 if (!strcmp(tmp->uuid, oa_config->uuid)) { 2394 drm_dbg(&oa->xe->drm, "OA config already exists with this uuid\n"); 2395 err = -EADDRINUSE; 2396 goto sysfs_err; 2397 } 2398 } 2399 2400 err = create_dynamic_oa_sysfs_entry(oa, oa_config); 2401 if (err) { 2402 drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n"); 2403 goto sysfs_err; 2404 } 2405 2406 oa_config->id = idr_alloc(&oa->metrics_idr, oa_config, 1, 0, GFP_KERNEL); 2407 if (oa_config->id < 0) { 2408 drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n"); 2409 err = oa_config->id; 2410 goto sysfs_err; 2411 } 2412 2413 id = oa_config->id; 2414 2415 drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, id); 2416 2417 mutex_unlock(&oa->metrics_lock); 2418 2419 return id; 2420 2421sysfs_err: 2422 mutex_unlock(&oa->metrics_lock); 2423reg_err: 2424 xe_oa_config_put(oa_config); 2425 drm_dbg(&oa->xe->drm, "Failed to add new OA config\n"); 2426 return err; 2427} 2428 2429/** 2430 * xe_oa_remove_config_ioctl - Removes one OA config 2431 * @dev: @drm_device 2432 * @data: pointer to struct @drm_xe_observation_param 2433 * @file: @drm_file 2434 */ 2435int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file) 2436{ 2437 struct xe_device *xe = to_xe_device(dev); 2438 struct xe_oa *oa = &xe->oa; 2439 struct xe_oa_config *oa_config; 2440 u64 arg, *ptr = u64_to_user_ptr(data); 2441 int ret; 2442 2443 if (!oa->xe) { 2444 drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); 2445 return -ENODEV; 2446 } 2447 2448 if (xe_observation_paranoid && !perfmon_capable()) { 2449 drm_dbg(&oa->xe->drm, "Insufficient privileges to remove xe OA config\n"); 2450 return -EACCES; 2451 } 2452 2453 ret = get_user(arg, ptr); 2454 if (XE_IOCTL_DBG(oa->xe, ret)) 2455 return ret; 2456 2457 ret = mutex_lock_interruptible(&oa->metrics_lock); 2458 if (ret) 2459 return ret; 2460 2461 oa_config = idr_find(&oa->metrics_idr, arg); 2462 if (!oa_config) { 2463 drm_dbg(&oa->xe->drm, "Failed to remove unknown OA config\n"); 2464 ret = -ENOENT; 2465 goto err_unlock; 2466 } 2467 2468 WARN_ON(arg != oa_config->id); 2469 2470 sysfs_remove_group(oa->metrics_kobj, &oa_config->sysfs_metric); 2471 idr_remove(&oa->metrics_idr, arg); 2472 2473 mutex_unlock(&oa->metrics_lock); 2474 2475 drm_dbg(&oa->xe->drm, "Removed config %s id=%i\n", oa_config->uuid, oa_config->id); 2476 2477 xe_oa_config_put(oa_config); 2478 2479 return 0; 2480 2481err_unlock: 2482 mutex_unlock(&oa->metrics_lock); 2483 return ret; 2484} 2485 2486static void xe_oa_unregister(void *arg) 2487{ 2488 struct xe_oa *oa = arg; 2489 2490 if (!oa->metrics_kobj) 2491 return; 2492 2493 kobject_put(oa->metrics_kobj); 2494 oa->metrics_kobj = NULL; 2495} 2496 2497/** 2498 * xe_oa_register - Xe OA registration 2499 * @xe: @xe_device 2500 * 2501 * Exposes the metrics sysfs directory upon completion of module initialization 2502 */ 2503int xe_oa_register(struct xe_device *xe) 2504{ 2505 struct xe_oa *oa = &xe->oa; 2506 2507 if (!oa->xe) 2508 return 0; 2509 2510 oa->metrics_kobj = kobject_create_and_add("metrics", 2511 &xe->drm.primary->kdev->kobj); 2512 if (!oa->metrics_kobj) 2513 return -ENOMEM; 2514 2515 return devm_add_action_or_reset(xe->drm.dev, xe_oa_unregister, oa); 2516} 2517 2518static u32 num_oa_units_per_gt(struct xe_gt *gt) 2519{ 2520 if (xe_gt_is_main_type(gt) || GRAPHICS_VER(gt_to_xe(gt)) < 20) 2521 return 1; 2522 else if (!IS_DGFX(gt_to_xe(gt))) 2523 return XE_OAM_UNIT_SCMI_0 + 1; /* SAG + SCMI_0 */ 2524 else 2525 return XE_OAM_UNIT_SCMI_1 + 1; /* SAG + SCMI_0 + SCMI_1 */ 2526} 2527 2528static u32 __hwe_oam_unit(struct xe_hw_engine *hwe) 2529{ 2530 if (GRAPHICS_VERx100(gt_to_xe(hwe->gt)) < 1270) 2531 return XE_OA_UNIT_INVALID; 2532 2533 xe_gt_WARN_ON(hwe->gt, xe_gt_is_main_type(hwe->gt)); 2534 2535 if (GRAPHICS_VER(gt_to_xe(hwe->gt)) < 20) 2536 return 0; 2537 /* 2538 * XE_OAM_UNIT_SAG has only GSCCS attached to it, but only on some platforms. Also 2539 * GSCCS cannot be used to submit batches to program the OAM unit. Therefore we don't 2540 * assign an OA unit to GSCCS. This means that XE_OAM_UNIT_SAG is exposed as an OA 2541 * unit without attached engines. Fused off engines can also result in oa_unit's with 2542 * num_engines == 0. OA streams can be opened on all OA units. 2543 */ 2544 else if (hwe->engine_id == XE_HW_ENGINE_GSCCS0) 2545 return XE_OA_UNIT_INVALID; 2546 else if (!IS_DGFX(gt_to_xe(hwe->gt))) 2547 return XE_OAM_UNIT_SCMI_0; 2548 else if (hwe->class == XE_ENGINE_CLASS_VIDEO_DECODE) 2549 return (hwe->instance / 2 & 0x1) + 1; 2550 else if (hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE) 2551 return (hwe->instance & 0x1) + 1; 2552 2553 return XE_OA_UNIT_INVALID; 2554} 2555 2556static u32 __hwe_oa_unit(struct xe_hw_engine *hwe) 2557{ 2558 switch (hwe->class) { 2559 case XE_ENGINE_CLASS_RENDER: 2560 case XE_ENGINE_CLASS_COMPUTE: 2561 return 0; 2562 2563 case XE_ENGINE_CLASS_VIDEO_DECODE: 2564 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 2565 case XE_ENGINE_CLASS_OTHER: 2566 return __hwe_oam_unit(hwe); 2567 2568 default: 2569 return XE_OA_UNIT_INVALID; 2570 } 2571} 2572 2573static struct xe_oa_regs __oam_regs(u32 base) 2574{ 2575 return (struct xe_oa_regs) { 2576 base, 2577 OAM_HEAD_POINTER(base), 2578 OAM_TAIL_POINTER(base), 2579 OAM_BUFFER(base), 2580 OAM_CONTEXT_CONTROL(base), 2581 OAM_CONTROL(base), 2582 OAM_DEBUG(base), 2583 OAM_STATUS(base), 2584 OAM_CONTROL_COUNTER_SEL_MASK, 2585 }; 2586} 2587 2588static struct xe_oa_regs __oag_regs(void) 2589{ 2590 return (struct xe_oa_regs) { 2591 0, 2592 OAG_OAHEADPTR, 2593 OAG_OATAILPTR, 2594 OAG_OABUFFER, 2595 OAG_OAGLBCTXCTRL, 2596 OAG_OACONTROL, 2597 OAG_OA_DEBUG, 2598 OAG_OASTATUS, 2599 OAG_OACONTROL_OA_COUNTER_SEL_MASK, 2600 }; 2601} 2602 2603static void __xe_oa_init_oa_units(struct xe_gt *gt) 2604{ 2605 /* Actual address is MEDIA_GT_GSI_OFFSET + oam_base_addr[i] */ 2606 const u32 oam_base_addr[] = { 2607 [XE_OAM_UNIT_SAG] = 0x13000, 2608 [XE_OAM_UNIT_SCMI_0] = 0x14000, 2609 [XE_OAM_UNIT_SCMI_1] = 0x14800, 2610 }; 2611 int i, num_units = gt->oa.num_oa_units; 2612 2613 for (i = 0; i < num_units; i++) { 2614 struct xe_oa_unit *u = >->oa.oa_unit[i]; 2615 2616 if (xe_gt_is_main_type(gt)) { 2617 u->regs = __oag_regs(); 2618 u->type = DRM_XE_OA_UNIT_TYPE_OAG; 2619 } else { 2620 xe_gt_assert(gt, GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270); 2621 u->regs = __oam_regs(oam_base_addr[i]); 2622 u->type = i == XE_OAM_UNIT_SAG && GRAPHICS_VER(gt_to_xe(gt)) >= 20 ? 2623 DRM_XE_OA_UNIT_TYPE_OAM_SAG : DRM_XE_OA_UNIT_TYPE_OAM; 2624 } 2625 2626 u->gt = gt; 2627 2628 xe_mmio_write32(>->mmio, u->regs.oa_ctrl, 0); 2629 2630 /* Ensure MMIO trigger remains disabled till there is a stream */ 2631 xe_mmio_write32(>->mmio, u->regs.oa_debug, 2632 oag_configure_mmio_trigger(NULL, false)); 2633 2634 /* Set oa_unit_ids now to ensure ids remain contiguous */ 2635 u->oa_unit_id = gt_to_xe(gt)->oa.oa_unit_ids++; 2636 } 2637} 2638 2639static int xe_oa_init_gt(struct xe_gt *gt) 2640{ 2641 u32 num_oa_units = num_oa_units_per_gt(gt); 2642 struct xe_hw_engine *hwe; 2643 enum xe_hw_engine_id id; 2644 struct xe_oa_unit *u; 2645 2646 u = drmm_kcalloc(>_to_xe(gt)->drm, num_oa_units, sizeof(*u), GFP_KERNEL); 2647 if (!u) 2648 return -ENOMEM; 2649 2650 for_each_hw_engine(hwe, gt, id) { 2651 u32 index = __hwe_oa_unit(hwe); 2652 2653 hwe->oa_unit = NULL; 2654 if (index < num_oa_units) { 2655 u[index].num_engines++; 2656 hwe->oa_unit = &u[index]; 2657 } 2658 } 2659 2660 gt->oa.num_oa_units = num_oa_units; 2661 gt->oa.oa_unit = u; 2662 2663 __xe_oa_init_oa_units(gt); 2664 2665 drmm_mutex_init(>_to_xe(gt)->drm, >->oa.gt_lock); 2666 2667 return 0; 2668} 2669 2670static void xe_oa_print_gt_oa_units(struct xe_gt *gt) 2671{ 2672 enum xe_hw_engine_id hwe_id; 2673 struct xe_hw_engine *hwe; 2674 struct xe_oa_unit *u; 2675 char buf[256]; 2676 int i, n; 2677 2678 for (i = 0; i < gt->oa.num_oa_units; i++) { 2679 u = >->oa.oa_unit[i]; 2680 buf[0] = '\0'; 2681 n = 0; 2682 2683 for_each_hw_engine(hwe, gt, hwe_id) 2684 if (xe_oa_unit_id(hwe) == u->oa_unit_id) 2685 n += scnprintf(buf + n, sizeof(buf) - n, "%s ", hwe->name); 2686 2687 xe_gt_dbg(gt, "oa_unit %d, type %d, Engines: %s\n", u->oa_unit_id, u->type, buf); 2688 } 2689} 2690 2691static void xe_oa_print_oa_units(struct xe_oa *oa) 2692{ 2693 struct xe_gt *gt; 2694 int gt_id; 2695 2696 for_each_gt(gt, oa->xe, gt_id) 2697 xe_oa_print_gt_oa_units(gt); 2698} 2699 2700static int xe_oa_init_oa_units(struct xe_oa *oa) 2701{ 2702 struct xe_gt *gt; 2703 int i, ret; 2704 2705 /* Needed for OAM implementation here */ 2706 BUILD_BUG_ON(XE_OAM_UNIT_SAG != 0); 2707 BUILD_BUG_ON(XE_OAM_UNIT_SCMI_0 != 1); 2708 BUILD_BUG_ON(XE_OAM_UNIT_SCMI_1 != 2); 2709 2710 for_each_gt(gt, oa->xe, i) { 2711 ret = xe_oa_init_gt(gt); 2712 if (ret) 2713 return ret; 2714 } 2715 2716 xe_oa_print_oa_units(oa); 2717 2718 return 0; 2719} 2720 2721static void oa_format_add(struct xe_oa *oa, enum xe_oa_format_name format) 2722{ 2723 __set_bit(format, oa->format_mask); 2724} 2725 2726static void xe_oa_init_supported_formats(struct xe_oa *oa) 2727{ 2728 if (GRAPHICS_VER(oa->xe) >= 20) { 2729 /* Xe2+ */ 2730 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8); 2731 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8); 2732 oa_format_add(oa, XE_OA_FORMAT_PEC64u64); 2733 oa_format_add(oa, XE_OA_FORMAT_PEC64u64_B8_C8); 2734 oa_format_add(oa, XE_OA_FORMAT_PEC64u32); 2735 oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G1); 2736 oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G1); 2737 oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G2); 2738 oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G2); 2739 oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_32_G2_4); 2740 oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_4_G2_32); 2741 } else if (GRAPHICS_VERx100(oa->xe) >= 1270) { 2742 /* XE_METEORLAKE */ 2743 oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8); 2744 oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8); 2745 oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8); 2746 oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8); 2747 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8); 2748 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8); 2749 } else if (GRAPHICS_VERx100(oa->xe) >= 1255) { 2750 /* XE_DG2, XE_PVC */ 2751 oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8); 2752 oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8); 2753 oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8); 2754 oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8); 2755 } else { 2756 /* Gen12+ */ 2757 xe_assert(oa->xe, GRAPHICS_VER(oa->xe) >= 12); 2758 oa_format_add(oa, XE_OA_FORMAT_A12); 2759 oa_format_add(oa, XE_OA_FORMAT_A12_B8_C8); 2760 oa_format_add(oa, XE_OA_FORMAT_A32u40_A4u32_B8_C8); 2761 oa_format_add(oa, XE_OA_FORMAT_C4_B8); 2762 } 2763} 2764 2765static int destroy_config(int id, void *p, void *data) 2766{ 2767 xe_oa_config_put(p); 2768 2769 return 0; 2770} 2771 2772static void xe_oa_fini(void *arg) 2773{ 2774 struct xe_device *xe = arg; 2775 struct xe_oa *oa = &xe->oa; 2776 2777 if (!oa->xe) 2778 return; 2779 2780 idr_for_each(&oa->metrics_idr, destroy_config, oa); 2781 idr_destroy(&oa->metrics_idr); 2782 2783 oa->xe = NULL; 2784} 2785 2786/** 2787 * xe_oa_init - OA initialization during device probe 2788 * @xe: @xe_device 2789 * 2790 * Return: 0 on success or a negative error code on failure 2791 */ 2792int xe_oa_init(struct xe_device *xe) 2793{ 2794 struct xe_oa *oa = &xe->oa; 2795 int ret; 2796 2797 /* Support OA only with GuC submission and Gen12+ */ 2798 if (!xe_device_uc_enabled(xe) || GRAPHICS_VER(xe) < 12) 2799 return 0; 2800 2801 if (IS_SRIOV_VF(xe)) 2802 return 0; 2803 2804 oa->xe = xe; 2805 oa->oa_formats = oa_formats; 2806 2807 drmm_mutex_init(&oa->xe->drm, &oa->metrics_lock); 2808 idr_init_base(&oa->metrics_idr, 1); 2809 2810 ret = xe_oa_init_oa_units(oa); 2811 if (ret) { 2812 drm_err(&xe->drm, "OA initialization failed (%pe)\n", ERR_PTR(ret)); 2813 goto exit; 2814 } 2815 2816 xe_oa_init_supported_formats(oa); 2817 2818 return devm_add_action_or_reset(xe->drm.dev, xe_oa_fini, xe); 2819 2820exit: 2821 oa->xe = NULL; 2822 return ret; 2823}
~/
Read-only file preview.