FILE
arm_brbe.c
File preview with syntax highlighting.
linux-6.19.14/drivers/perf/arm_brbe.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Branch Record Buffer Extension Driver. 4 * 5 * Copyright (C) 2022-2025 ARM Limited 6 * 7 * Author: Anshuman Khandual <anshuman.khandual@arm.com> 8 */ 9#include <linux/types.h> 10#include <linux/bitmap.h> 11#include <linux/perf/arm_pmu.h> 12#include "arm_brbe.h" 13 14#define BRBFCR_EL1_BRANCH_FILTERS (BRBFCR_EL1_DIRECT | \ 15 BRBFCR_EL1_INDIRECT | \ 16 BRBFCR_EL1_RTN | \ 17 BRBFCR_EL1_INDCALL | \ 18 BRBFCR_EL1_DIRCALL | \ 19 BRBFCR_EL1_CONDDIR) 20 21/* 22 * BRBTS_EL1 is currently not used for branch stack implementation 23 * purpose but BRBCR_ELx.TS needs to have a valid value from all 24 * available options. BRBCR_ELx_TS_VIRTUAL is selected for this. 25 */ 26#define BRBCR_ELx_DEFAULT_TS FIELD_PREP(BRBCR_ELx_TS_MASK, BRBCR_ELx_TS_VIRTUAL) 27 28/* 29 * BRBE Buffer Organization 30 * 31 * BRBE buffer is arranged as multiple banks of 32 branch record 32 * entries each. An individual branch record in a given bank could 33 * be accessed, after selecting the bank in BRBFCR_EL1.BANK and 34 * accessing the registers i.e [BRBSRC, BRBTGT, BRBINF] set with 35 * indices [0..31]. 36 * 37 * Bank 0 38 * 39 * --------------------------------- ------ 40 * | 00 | BRBSRC | BRBTGT | BRBINF | | 00 | 41 * --------------------------------- ------ 42 * | 01 | BRBSRC | BRBTGT | BRBINF | | 01 | 43 * --------------------------------- ------ 44 * | .. | BRBSRC | BRBTGT | BRBINF | | .. | 45 * --------------------------------- ------ 46 * | 31 | BRBSRC | BRBTGT | BRBINF | | 31 | 47 * --------------------------------- ------ 48 * 49 * Bank 1 50 * 51 * --------------------------------- ------ 52 * | 32 | BRBSRC | BRBTGT | BRBINF | | 00 | 53 * --------------------------------- ------ 54 * | 33 | BRBSRC | BRBTGT | BRBINF | | 01 | 55 * --------------------------------- ------ 56 * | .. | BRBSRC | BRBTGT | BRBINF | | .. | 57 * --------------------------------- ------ 58 * | 63 | BRBSRC | BRBTGT | BRBINF | | 31 | 59 * --------------------------------- ------ 60 */ 61#define BRBE_BANK_MAX_ENTRIES 32 62 63struct brbe_regset { 64 u64 brbsrc; 65 u64 brbtgt; 66 u64 brbinf; 67}; 68 69#define PERF_BR_ARM64_MAX (PERF_BR_MAX + PERF_BR_NEW_MAX) 70 71struct brbe_hw_attr { 72 int brbe_version; 73 int brbe_cc; 74 int brbe_nr; 75 int brbe_format; 76}; 77 78#define BRBE_REGN_CASE(n, case_macro) \ 79 case n: case_macro(n); break 80 81#define BRBE_REGN_SWITCH(x, case_macro) \ 82 do { \ 83 switch (x) { \ 84 BRBE_REGN_CASE(0, case_macro); \ 85 BRBE_REGN_CASE(1, case_macro); \ 86 BRBE_REGN_CASE(2, case_macro); \ 87 BRBE_REGN_CASE(3, case_macro); \ 88 BRBE_REGN_CASE(4, case_macro); \ 89 BRBE_REGN_CASE(5, case_macro); \ 90 BRBE_REGN_CASE(6, case_macro); \ 91 BRBE_REGN_CASE(7, case_macro); \ 92 BRBE_REGN_CASE(8, case_macro); \ 93 BRBE_REGN_CASE(9, case_macro); \ 94 BRBE_REGN_CASE(10, case_macro); \ 95 BRBE_REGN_CASE(11, case_macro); \ 96 BRBE_REGN_CASE(12, case_macro); \ 97 BRBE_REGN_CASE(13, case_macro); \ 98 BRBE_REGN_CASE(14, case_macro); \ 99 BRBE_REGN_CASE(15, case_macro); \ 100 BRBE_REGN_CASE(16, case_macro); \ 101 BRBE_REGN_CASE(17, case_macro); \ 102 BRBE_REGN_CASE(18, case_macro); \ 103 BRBE_REGN_CASE(19, case_macro); \ 104 BRBE_REGN_CASE(20, case_macro); \ 105 BRBE_REGN_CASE(21, case_macro); \ 106 BRBE_REGN_CASE(22, case_macro); \ 107 BRBE_REGN_CASE(23, case_macro); \ 108 BRBE_REGN_CASE(24, case_macro); \ 109 BRBE_REGN_CASE(25, case_macro); \ 110 BRBE_REGN_CASE(26, case_macro); \ 111 BRBE_REGN_CASE(27, case_macro); \ 112 BRBE_REGN_CASE(28, case_macro); \ 113 BRBE_REGN_CASE(29, case_macro); \ 114 BRBE_REGN_CASE(30, case_macro); \ 115 BRBE_REGN_CASE(31, case_macro); \ 116 default: WARN(1, "Invalid BRB* index %d\n", x); \ 117 } \ 118 } while (0) 119 120#define RETURN_READ_BRBSRCN(n) \ 121 return read_sysreg_s(SYS_BRBSRC_EL1(n)) 122static inline u64 get_brbsrc_reg(int idx) 123{ 124 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBSRCN); 125 return 0; 126} 127 128#define RETURN_READ_BRBTGTN(n) \ 129 return read_sysreg_s(SYS_BRBTGT_EL1(n)) 130static u64 get_brbtgt_reg(int idx) 131{ 132 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBTGTN); 133 return 0; 134} 135 136#define RETURN_READ_BRBINFN(n) \ 137 return read_sysreg_s(SYS_BRBINF_EL1(n)) 138static u64 get_brbinf_reg(int idx) 139{ 140 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBINFN); 141 return 0; 142} 143 144static u64 brbe_record_valid(u64 brbinf) 145{ 146 return FIELD_GET(BRBINFx_EL1_VALID_MASK, brbinf); 147} 148 149static bool brbe_invalid(u64 brbinf) 150{ 151 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_NONE; 152} 153 154static bool brbe_record_is_complete(u64 brbinf) 155{ 156 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_FULL; 157} 158 159static bool brbe_record_is_source_only(u64 brbinf) 160{ 161 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_SOURCE; 162} 163 164static bool brbe_record_is_target_only(u64 brbinf) 165{ 166 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_TARGET; 167} 168 169static int brbinf_get_in_tx(u64 brbinf) 170{ 171 return FIELD_GET(BRBINFx_EL1_T_MASK, brbinf); 172} 173 174static int brbinf_get_mispredict(u64 brbinf) 175{ 176 return FIELD_GET(BRBINFx_EL1_MPRED_MASK, brbinf); 177} 178 179static int brbinf_get_lastfailed(u64 brbinf) 180{ 181 return FIELD_GET(BRBINFx_EL1_LASTFAILED_MASK, brbinf); 182} 183 184static u16 brbinf_get_cycles(u64 brbinf) 185{ 186 u32 exp, mant, cycles; 187 /* 188 * Captured cycle count is unknown and hence 189 * should not be passed on to userspace. 190 */ 191 if (brbinf & BRBINFx_EL1_CCU) 192 return 0; 193 194 exp = FIELD_GET(BRBINFx_EL1_CC_EXP_MASK, brbinf); 195 mant = FIELD_GET(BRBINFx_EL1_CC_MANT_MASK, brbinf); 196 197 if (!exp) 198 return mant; 199 200 cycles = (mant | 0x100) << (exp - 1); 201 202 return min(cycles, U16_MAX); 203} 204 205static int brbinf_get_type(u64 brbinf) 206{ 207 return FIELD_GET(BRBINFx_EL1_TYPE_MASK, brbinf); 208} 209 210static int brbinf_get_el(u64 brbinf) 211{ 212 return FIELD_GET(BRBINFx_EL1_EL_MASK, brbinf); 213} 214 215void brbe_invalidate(void) 216{ 217 /* Ensure all branches before this point are recorded */ 218 isb(); 219 asm volatile(BRB_IALL_INSN); 220 /* Ensure all branch records are invalidated after this point */ 221 isb(); 222} 223 224static bool valid_brbe_nr(int brbe_nr) 225{ 226 return brbe_nr == BRBIDR0_EL1_NUMREC_8 || 227 brbe_nr == BRBIDR0_EL1_NUMREC_16 || 228 brbe_nr == BRBIDR0_EL1_NUMREC_32 || 229 brbe_nr == BRBIDR0_EL1_NUMREC_64; 230} 231 232static bool valid_brbe_cc(int brbe_cc) 233{ 234 return brbe_cc == BRBIDR0_EL1_CC_20_BIT; 235} 236 237static bool valid_brbe_format(int brbe_format) 238{ 239 return brbe_format == BRBIDR0_EL1_FORMAT_FORMAT_0; 240} 241 242static bool valid_brbidr(u64 brbidr) 243{ 244 int brbe_format, brbe_cc, brbe_nr; 245 246 brbe_format = FIELD_GET(BRBIDR0_EL1_FORMAT_MASK, brbidr); 247 brbe_cc = FIELD_GET(BRBIDR0_EL1_CC_MASK, brbidr); 248 brbe_nr = FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, brbidr); 249 250 return valid_brbe_format(brbe_format) && valid_brbe_cc(brbe_cc) && valid_brbe_nr(brbe_nr); 251} 252 253static bool valid_brbe_version(int brbe_version) 254{ 255 return brbe_version == ID_AA64DFR0_EL1_BRBE_IMP || 256 brbe_version == ID_AA64DFR0_EL1_BRBE_BRBE_V1P1; 257} 258 259static void select_brbe_bank(int bank) 260{ 261 u64 brbfcr; 262 263 brbfcr = read_sysreg_s(SYS_BRBFCR_EL1); 264 brbfcr &= ~BRBFCR_EL1_BANK_MASK; 265 brbfcr |= SYS_FIELD_PREP(BRBFCR_EL1, BANK, bank); 266 write_sysreg_s(brbfcr, SYS_BRBFCR_EL1); 267 /* 268 * Arm ARM (DDI 0487K.a) D.18.4 rule PPBZP requires explicit sync 269 * between setting BANK and accessing branch records. 270 */ 271 isb(); 272} 273 274static bool __read_brbe_regset(struct brbe_regset *entry, int idx) 275{ 276 entry->brbinf = get_brbinf_reg(idx); 277 278 if (brbe_invalid(entry->brbinf)) 279 return false; 280 281 entry->brbsrc = get_brbsrc_reg(idx); 282 entry->brbtgt = get_brbtgt_reg(idx); 283 return true; 284} 285 286/* 287 * Generic perf branch filters supported on BRBE 288 * 289 * New branch filters need to be evaluated whether they could be supported on 290 * BRBE. This ensures that such branch filters would not just be accepted, to 291 * fail silently. PERF_SAMPLE_BRANCH_HV is a special case that is selectively 292 * supported only on platforms where kernel is in hyp mode. 293 */ 294#define BRBE_EXCLUDE_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_ABORT_TX | \ 295 PERF_SAMPLE_BRANCH_IN_TX | \ 296 PERF_SAMPLE_BRANCH_NO_TX | \ 297 PERF_SAMPLE_BRANCH_CALL_STACK | \ 298 PERF_SAMPLE_BRANCH_COUNTERS) 299 300#define BRBE_ALLOWED_BRANCH_TYPES (PERF_SAMPLE_BRANCH_ANY | \ 301 PERF_SAMPLE_BRANCH_ANY_CALL | \ 302 PERF_SAMPLE_BRANCH_ANY_RETURN | \ 303 PERF_SAMPLE_BRANCH_IND_CALL | \ 304 PERF_SAMPLE_BRANCH_COND | \ 305 PERF_SAMPLE_BRANCH_IND_JUMP | \ 306 PERF_SAMPLE_BRANCH_CALL) 307 308 309#define BRBE_ALLOWED_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_USER | \ 310 PERF_SAMPLE_BRANCH_KERNEL | \ 311 PERF_SAMPLE_BRANCH_HV | \ 312 BRBE_ALLOWED_BRANCH_TYPES | \ 313 PERF_SAMPLE_BRANCH_NO_FLAGS | \ 314 PERF_SAMPLE_BRANCH_NO_CYCLES | \ 315 PERF_SAMPLE_BRANCH_TYPE_SAVE | \ 316 PERF_SAMPLE_BRANCH_HW_INDEX | \ 317 PERF_SAMPLE_BRANCH_PRIV_SAVE) 318 319#define BRBE_PERF_BRANCH_FILTERS (BRBE_ALLOWED_BRANCH_FILTERS | \ 320 BRBE_EXCLUDE_BRANCH_FILTERS) 321 322/* 323 * BRBE supports the following functional branch type filters while 324 * generating branch records. These branch filters can be enabled, 325 * either individually or as a group i.e ORing multiple filters 326 * with each other. 327 * 328 * BRBFCR_EL1_CONDDIR - Conditional direct branch 329 * BRBFCR_EL1_DIRCALL - Direct call 330 * BRBFCR_EL1_INDCALL - Indirect call 331 * BRBFCR_EL1_INDIRECT - Indirect branch 332 * BRBFCR_EL1_DIRECT - Direct branch 333 * BRBFCR_EL1_RTN - Subroutine return 334 */ 335static u64 branch_type_to_brbfcr(int branch_type) 336{ 337 u64 brbfcr = 0; 338 339 if (branch_type & PERF_SAMPLE_BRANCH_ANY) { 340 brbfcr |= BRBFCR_EL1_BRANCH_FILTERS; 341 return brbfcr; 342 } 343 344 if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) { 345 brbfcr |= BRBFCR_EL1_INDCALL; 346 brbfcr |= BRBFCR_EL1_DIRCALL; 347 } 348 349 if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 350 brbfcr |= BRBFCR_EL1_RTN; 351 352 if (branch_type & PERF_SAMPLE_BRANCH_IND_CALL) 353 brbfcr |= BRBFCR_EL1_INDCALL; 354 355 if (branch_type & PERF_SAMPLE_BRANCH_COND) 356 brbfcr |= BRBFCR_EL1_CONDDIR; 357 358 if (branch_type & PERF_SAMPLE_BRANCH_IND_JUMP) 359 brbfcr |= BRBFCR_EL1_INDIRECT; 360 361 if (branch_type & PERF_SAMPLE_BRANCH_CALL) 362 brbfcr |= BRBFCR_EL1_DIRCALL; 363 364 return brbfcr; 365} 366 367/* 368 * BRBE supports the following privilege mode filters while generating 369 * branch records. 370 * 371 * BRBCR_ELx_E0BRE - EL0 branch records 372 * BRBCR_ELx_ExBRE - EL1/EL2 branch records 373 * 374 * BRBE also supports the following additional functional branch type 375 * filters while generating branch records. 376 * 377 * BRBCR_ELx_EXCEPTION - Exception 378 * BRBCR_ELx_ERTN - Exception return 379 */ 380static u64 branch_type_to_brbcr(int branch_type) 381{ 382 u64 brbcr = BRBCR_ELx_FZP | BRBCR_ELx_DEFAULT_TS; 383 384 if (branch_type & PERF_SAMPLE_BRANCH_USER) 385 brbcr |= BRBCR_ELx_E0BRE; 386 387 /* 388 * When running in the hyp mode, writing into BRBCR_EL1 389 * actually writes into BRBCR_EL2 instead. Field E2BRE 390 * is also at the same position as E1BRE. 391 */ 392 if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) 393 brbcr |= BRBCR_ELx_ExBRE; 394 395 if (branch_type & PERF_SAMPLE_BRANCH_HV) { 396 if (is_kernel_in_hyp_mode()) 397 brbcr |= BRBCR_ELx_ExBRE; 398 } 399 400 if (!(branch_type & PERF_SAMPLE_BRANCH_NO_CYCLES)) 401 brbcr |= BRBCR_ELx_CC; 402 403 if (!(branch_type & PERF_SAMPLE_BRANCH_NO_FLAGS)) 404 brbcr |= BRBCR_ELx_MPRED; 405 406 /* 407 * The exception and exception return branches could be 408 * captured, irrespective of the perf event's privilege. 409 * If the perf event does not have enough privilege for 410 * a given exception level, then addresses which falls 411 * under that exception level will be reported as zero 412 * for the captured branch record, creating source only 413 * or target only records. 414 */ 415 if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) { 416 if (branch_type & PERF_SAMPLE_BRANCH_ANY) { 417 brbcr |= BRBCR_ELx_EXCEPTION; 418 brbcr |= BRBCR_ELx_ERTN; 419 } 420 421 if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) 422 brbcr |= BRBCR_ELx_EXCEPTION; 423 424 if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 425 brbcr |= BRBCR_ELx_ERTN; 426 } 427 return brbcr; 428} 429 430bool brbe_branch_attr_valid(struct perf_event *event) 431{ 432 u64 branch_type = event->attr.branch_sample_type; 433 434 /* 435 * Ensure both perf branch filter allowed and exclude 436 * masks are always in sync with the generic perf ABI. 437 */ 438 BUILD_BUG_ON(BRBE_PERF_BRANCH_FILTERS != (PERF_SAMPLE_BRANCH_MAX - 1)); 439 440 if (branch_type & BRBE_EXCLUDE_BRANCH_FILTERS) { 441 pr_debug("requested branch filter not supported 0x%llx\n", branch_type); 442 return false; 443 } 444 445 /* Ensure at least 1 branch type is enabled */ 446 if (!(branch_type & BRBE_ALLOWED_BRANCH_TYPES)) { 447 pr_debug("no branch type enabled 0x%llx\n", branch_type); 448 return false; 449 } 450 451 /* 452 * No branches are recorded in guests nor nVHE hypervisors, so 453 * excluding the host or both kernel and user is invalid. 454 * 455 * Ideally we'd just require exclude_guest and exclude_hv, but setting 456 * event filters with perf for kernel or user don't set exclude_guest. 457 * So effectively, exclude_guest and exclude_hv are ignored. 458 */ 459 if (event->attr.exclude_host || (event->attr.exclude_user && event->attr.exclude_kernel)) { 460 pr_debug("branch filter in hypervisor or guest only not supported 0x%llx\n", branch_type); 461 return false; 462 } 463 464 event->hw.branch_reg.config = branch_type_to_brbfcr(event->attr.branch_sample_type); 465 event->hw.extra_reg.config = branch_type_to_brbcr(event->attr.branch_sample_type); 466 467 return true; 468} 469 470unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu) 471{ 472 return FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, armpmu->reg_brbidr); 473} 474 475void brbe_probe(struct arm_pmu *armpmu) 476{ 477 u64 brbidr, aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); 478 u32 brbe; 479 480 brbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT); 481 if (!valid_brbe_version(brbe)) 482 return; 483 484 brbidr = read_sysreg_s(SYS_BRBIDR0_EL1); 485 if (!valid_brbidr(brbidr)) 486 return; 487 488 armpmu->reg_brbidr = brbidr; 489} 490 491/* 492 * BRBE is assumed to be disabled/paused on entry 493 */ 494void brbe_enable(const struct arm_pmu *arm_pmu) 495{ 496 struct pmu_hw_events *cpuc = this_cpu_ptr(arm_pmu->hw_events); 497 u64 brbfcr = 0, brbcr = 0; 498 499 /* 500 * Discard existing records to avoid a discontinuity, e.g. records 501 * missed during handling an overflow. 502 */ 503 brbe_invalidate(); 504 505 /* 506 * Merge the permitted branch filters of all events. 507 */ 508 for (int i = 0; i < ARMPMU_MAX_HWEVENTS; i++) { 509 struct perf_event *event = cpuc->events[i]; 510 511 if (event && has_branch_stack(event)) { 512 brbfcr |= event->hw.branch_reg.config; 513 brbcr |= event->hw.extra_reg.config; 514 } 515 } 516 517 /* 518 * In VHE mode with MDCR_EL2.HPMN equal to PMCR_EL0.N, BRBCR_EL1.FZP 519 * controls freezing the branch records on counter overflow rather than 520 * BRBCR_EL2.FZP (which writes to BRBCR_EL1 are redirected to). 521 * The exception levels are enabled/disabled in BRBCR_EL2, so keep EL1 522 * and EL0 recording disabled for guests. 523 * 524 * As BRBCR_EL1 CC and MPRED bits also need to match, use the same 525 * value for both registers just masking the exception levels. 526 */ 527 if (is_kernel_in_hyp_mode()) 528 write_sysreg_s(brbcr & ~(BRBCR_ELx_ExBRE | BRBCR_ELx_E0BRE), SYS_BRBCR_EL12); 529 write_sysreg_s(brbcr, SYS_BRBCR_EL1); 530 /* Ensure BRBCR_ELx settings take effect before unpausing */ 531 isb(); 532 533 /* Finally write SYS_BRBFCR_EL to unpause BRBE */ 534 write_sysreg_s(brbfcr, SYS_BRBFCR_EL1); 535 /* Synchronization in PMCR write ensures ordering WRT PMU enabling */ 536} 537 538void brbe_disable(void) 539{ 540 /* 541 * No need for synchronization here as synchronization in PMCR write 542 * ensures ordering and in the interrupt handler this is a NOP as 543 * we're already paused. 544 */ 545 write_sysreg_s(BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1); 546 write_sysreg_s(0, SYS_BRBCR_EL1); 547} 548 549static const int brbe_type_to_perf_type_map[BRBINFx_EL1_TYPE_DEBUG_EXIT + 1][2] = { 550 [BRBINFx_EL1_TYPE_DIRECT_UNCOND] = { PERF_BR_UNCOND, 0 }, 551 [BRBINFx_EL1_TYPE_INDIRECT] = { PERF_BR_IND, 0 }, 552 [BRBINFx_EL1_TYPE_DIRECT_LINK] = { PERF_BR_CALL, 0 }, 553 [BRBINFx_EL1_TYPE_INDIRECT_LINK] = { PERF_BR_IND_CALL, 0 }, 554 [BRBINFx_EL1_TYPE_RET] = { PERF_BR_RET, 0 }, 555 [BRBINFx_EL1_TYPE_DIRECT_COND] = { PERF_BR_COND, 0 }, 556 [BRBINFx_EL1_TYPE_CALL] = { PERF_BR_SYSCALL, 0 }, 557 [BRBINFx_EL1_TYPE_ERET] = { PERF_BR_ERET, 0 }, 558 [BRBINFx_EL1_TYPE_IRQ] = { PERF_BR_IRQ, 0 }, 559 [BRBINFx_EL1_TYPE_TRAP] = { PERF_BR_IRQ, 0 }, 560 [BRBINFx_EL1_TYPE_SERROR] = { PERF_BR_SERROR, 0 }, 561 [BRBINFx_EL1_TYPE_ALIGN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_ALGN }, 562 [BRBINFx_EL1_TYPE_INSN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_INST }, 563 [BRBINFx_EL1_TYPE_DATA_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_DATA }, 564}; 565 566static void brbe_set_perf_entry_type(struct perf_branch_entry *entry, u64 brbinf) 567{ 568 int brbe_type = brbinf_get_type(brbinf); 569 570 if (brbe_type <= BRBINFx_EL1_TYPE_DEBUG_EXIT) { 571 const int *br_type = brbe_type_to_perf_type_map[brbe_type]; 572 573 entry->type = br_type[0]; 574 entry->new_type = br_type[1]; 575 } 576} 577 578static int brbinf_get_perf_priv(u64 brbinf) 579{ 580 int brbe_el = brbinf_get_el(brbinf); 581 582 switch (brbe_el) { 583 case BRBINFx_EL1_EL_EL0: 584 return PERF_BR_PRIV_USER; 585 case BRBINFx_EL1_EL_EL1: 586 return PERF_BR_PRIV_KERNEL; 587 case BRBINFx_EL1_EL_EL2: 588 if (is_kernel_in_hyp_mode()) 589 return PERF_BR_PRIV_KERNEL; 590 return PERF_BR_PRIV_HV; 591 default: 592 pr_warn_once("%d - unknown branch privilege captured\n", brbe_el); 593 return PERF_BR_PRIV_UNKNOWN; 594 } 595} 596 597static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *entry, 598 const struct perf_event *event) 599{ 600 struct brbe_regset bregs; 601 u64 brbinf; 602 603 if (!__read_brbe_regset(&bregs, index)) 604 return false; 605 606 brbinf = bregs.brbinf; 607 perf_clear_branch_entry_bitfields(entry); 608 if (brbe_record_is_complete(brbinf)) { 609 entry->from = bregs.brbsrc; 610 entry->to = bregs.brbtgt; 611 } else if (brbe_record_is_source_only(brbinf)) { 612 entry->from = bregs.brbsrc; 613 entry->to = 0; 614 } else if (brbe_record_is_target_only(brbinf)) { 615 entry->from = 0; 616 entry->to = bregs.brbtgt; 617 } 618 619 brbe_set_perf_entry_type(entry, brbinf); 620 621 if (!branch_sample_no_cycles(event)) 622 entry->cycles = brbinf_get_cycles(brbinf); 623 624 if (!branch_sample_no_flags(event)) { 625 /* Mispredict info is available for source only and complete branch records. */ 626 if (!brbe_record_is_target_only(brbinf)) { 627 entry->mispred = brbinf_get_mispredict(brbinf); 628 entry->predicted = !entry->mispred; 629 } 630 631 /* 632 * Currently TME feature is neither implemented in any hardware 633 * nor it is being supported in the kernel. Just warn here once 634 * if TME related information shows up rather unexpectedly. 635 */ 636 if (brbinf_get_lastfailed(brbinf) || brbinf_get_in_tx(brbinf)) 637 pr_warn_once("Unknown transaction states\n"); 638 } 639 640 /* 641 * Branch privilege level is available for target only and complete 642 * branch records. 643 */ 644 if (!brbe_record_is_source_only(brbinf)) 645 entry->priv = brbinf_get_perf_priv(brbinf); 646 647 return true; 648} 649 650#define PERF_BR_ARM64_ALL ( \ 651 BIT(PERF_BR_COND) | \ 652 BIT(PERF_BR_UNCOND) | \ 653 BIT(PERF_BR_IND) | \ 654 BIT(PERF_BR_CALL) | \ 655 BIT(PERF_BR_IND_CALL) | \ 656 BIT(PERF_BR_RET)) 657 658#define PERF_BR_ARM64_ALL_KERNEL ( \ 659 BIT(PERF_BR_SYSCALL) | \ 660 BIT(PERF_BR_IRQ) | \ 661 BIT(PERF_BR_SERROR) | \ 662 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_ALGN) | \ 663 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_DATA) | \ 664 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_INST)) 665 666static void prepare_event_branch_type_mask(u64 branch_sample, 667 unsigned long *event_type_mask) 668{ 669 if (branch_sample & PERF_SAMPLE_BRANCH_ANY) { 670 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) 671 bitmap_from_u64(event_type_mask, 672 BIT(PERF_BR_ERET) | PERF_BR_ARM64_ALL | 673 PERF_BR_ARM64_ALL_KERNEL); 674 else 675 bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL); 676 return; 677 } 678 679 bitmap_zero(event_type_mask, PERF_BR_ARM64_MAX); 680 681 if (branch_sample & PERF_SAMPLE_BRANCH_ANY_CALL) { 682 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) 683 bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL_KERNEL); 684 685 set_bit(PERF_BR_CALL, event_type_mask); 686 set_bit(PERF_BR_IND_CALL, event_type_mask); 687 } 688 689 if (branch_sample & PERF_SAMPLE_BRANCH_IND_JUMP) 690 set_bit(PERF_BR_IND, event_type_mask); 691 692 if (branch_sample & PERF_SAMPLE_BRANCH_COND) 693 set_bit(PERF_BR_COND, event_type_mask); 694 695 if (branch_sample & PERF_SAMPLE_BRANCH_CALL) 696 set_bit(PERF_BR_CALL, event_type_mask); 697 698 if (branch_sample & PERF_SAMPLE_BRANCH_IND_CALL) 699 set_bit(PERF_BR_IND_CALL, event_type_mask); 700 701 if (branch_sample & PERF_SAMPLE_BRANCH_ANY_RETURN) { 702 set_bit(PERF_BR_RET, event_type_mask); 703 704 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) 705 set_bit(PERF_BR_ERET, event_type_mask); 706 } 707} 708 709/* 710 * BRBE is configured with an OR of permissions from all events, so there may 711 * be events which have to be dropped or events where just the source or target 712 * address has to be zeroed. 713 */ 714static bool filter_branch_privilege(struct perf_branch_entry *entry, u64 branch_sample_type) 715{ 716 bool from_user = access_ok((void __user *)(unsigned long)entry->from, 4); 717 bool to_user = access_ok((void __user *)(unsigned long)entry->to, 4); 718 bool exclude_kernel = !((branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) || 719 (is_kernel_in_hyp_mode() && (branch_sample_type & PERF_SAMPLE_BRANCH_HV))); 720 721 /* We can only have a half record if permissions have not been expanded */ 722 if (!entry->from || !entry->to) 723 return true; 724 725 /* 726 * If record is within a single exception level, just need to either 727 * drop or keep the entire record. 728 */ 729 if (from_user == to_user) 730 return ((entry->priv == PERF_BR_PRIV_KERNEL) && !exclude_kernel) || 731 ((entry->priv == PERF_BR_PRIV_USER) && 732 (branch_sample_type & PERF_SAMPLE_BRANCH_USER)); 733 734 /* 735 * Record is across exception levels, mask addresses for the exception 736 * level we're not capturing. 737 */ 738 if (!(branch_sample_type & PERF_SAMPLE_BRANCH_USER)) { 739 if (from_user) 740 entry->from = 0; 741 if (to_user) 742 entry->to = 0; 743 } 744 745 if (exclude_kernel) { 746 if (!from_user) 747 entry->from = 0; 748 if (!to_user) 749 entry->to = 0; 750 } 751 752 return true; 753} 754 755static bool filter_branch_type(struct perf_branch_entry *entry, 756 const unsigned long *event_type_mask) 757{ 758 if (entry->type == PERF_BR_EXTEND_ABI) 759 return test_bit(PERF_BR_MAX + entry->new_type, event_type_mask); 760 else 761 return test_bit(entry->type, event_type_mask); 762} 763 764static bool filter_branch_record(struct perf_branch_entry *entry, 765 u64 branch_sample, 766 const unsigned long *event_type_mask) 767{ 768 return filter_branch_type(entry, event_type_mask) && 769 filter_branch_privilege(entry, branch_sample); 770} 771 772void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack, 773 const struct perf_event *event) 774{ 775 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 776 int nr_hw = brbe_num_branch_records(cpu_pmu); 777 int nr_banks = DIV_ROUND_UP(nr_hw, BRBE_BANK_MAX_ENTRIES); 778 int nr_filtered = 0; 779 u64 branch_sample_type = event->attr.branch_sample_type; 780 DECLARE_BITMAP(event_type_mask, PERF_BR_ARM64_MAX); 781 782 prepare_event_branch_type_mask(branch_sample_type, event_type_mask); 783 784 for (int bank = 0; bank < nr_banks; bank++) { 785 int nr_remaining = nr_hw - (bank * BRBE_BANK_MAX_ENTRIES); 786 int nr_this_bank = min(nr_remaining, BRBE_BANK_MAX_ENTRIES); 787 788 select_brbe_bank(bank); 789 790 for (int i = 0; i < nr_this_bank; i++) { 791 struct perf_branch_entry *pbe = &branch_stack->entries[nr_filtered]; 792 793 if (!perf_entry_from_brbe_regset(i, pbe, event)) 794 goto done; 795 796 if (!filter_branch_record(pbe, branch_sample_type, event_type_mask)) 797 continue; 798 799 nr_filtered++; 800 } 801 } 802 803done: 804 branch_stack->nr = nr_filtered; 805}
~/
Read-only file preview.