FILE
blktrace.c
File preview with syntax highlighting.
linux-6.19.14/kernel/trace/blktrace.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk> 4 * 5 */ 6 7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9#include <linux/kernel.h> 10#include <linux/blkdev.h> 11#include <linux/blktrace_api.h> 12#include <linux/percpu.h> 13#include <linux/init.h> 14#include <linux/mutex.h> 15#include <linux/slab.h> 16#include <linux/debugfs.h> 17#include <linux/export.h> 18#include <linux/time.h> 19#include <linux/uaccess.h> 20#include <linux/list.h> 21#include <linux/blk-cgroup.h> 22 23#include "../../block/blk.h" 24 25#include <trace/events/block.h> 26 27#include "trace_output.h" 28 29#ifdef CONFIG_BLK_DEV_IO_TRACE 30 31static unsigned int blktrace_seq __read_mostly = 1; 32 33static struct trace_array *blk_tr; 34static bool blk_tracer_enabled __read_mostly; 35 36static LIST_HEAD(running_trace_list); 37static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(running_trace_lock); 38 39/* Select an alternative, minimalistic output than the original one */ 40#define TRACE_BLK_OPT_CLASSIC 0x1 41#define TRACE_BLK_OPT_CGROUP 0x2 42#define TRACE_BLK_OPT_CGNAME 0x4 43 44static struct tracer_opt blk_tracer_opts[] = { 45 /* Default disable the minimalistic output */ 46 { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) }, 47#ifdef CONFIG_BLK_CGROUP 48 { TRACER_OPT(blk_cgroup, TRACE_BLK_OPT_CGROUP) }, 49 { TRACER_OPT(blk_cgname, TRACE_BLK_OPT_CGNAME) }, 50#endif 51 { } 52}; 53 54static struct tracer_flags blk_tracer_flags = { 55 .val = 0, 56 .opts = blk_tracer_opts, 57}; 58 59/* Global reference count of probes */ 60static DEFINE_MUTEX(blk_probe_mutex); 61static int blk_probes_ref; 62 63static void blk_register_tracepoints(void); 64static void blk_unregister_tracepoints(void); 65 66static void record_blktrace_event(struct blk_io_trace *t, pid_t pid, int cpu, 67 sector_t sector, int bytes, u64 what, 68 dev_t dev, int error, u64 cgid, 69 ssize_t cgid_len, void *pdu_data, int pdu_len) 70 71{ 72 /* 73 * These two are not needed in ftrace as they are in the 74 * generic trace_entry, filled by tracing_generic_entry_update, 75 * but for the trace_event->bin() synthesizer benefit we do it 76 * here too. 77 */ 78 t->cpu = cpu; 79 t->pid = pid; 80 81 t->sector = sector; 82 t->bytes = bytes; 83 t->action = lower_32_bits(what); 84 t->device = dev; 85 t->error = error; 86 t->pdu_len = pdu_len + cgid_len; 87 88 if (cgid_len) 89 memcpy((void *)t + sizeof(*t), &cgid, cgid_len); 90 if (pdu_len) 91 memcpy((void *)t + sizeof(*t) + cgid_len, pdu_data, pdu_len); 92} 93 94static void record_blktrace_event2(struct blk_io_trace2 *t2, pid_t pid, int cpu, 95 sector_t sector, int bytes, u64 what, 96 dev_t dev, int error, u64 cgid, 97 ssize_t cgid_len, void *pdu_data, 98 int pdu_len) 99{ 100 t2->pid = pid; 101 t2->cpu = cpu; 102 103 t2->sector = sector; 104 t2->bytes = bytes; 105 t2->action = what; 106 t2->device = dev; 107 t2->error = error; 108 t2->pdu_len = pdu_len + cgid_len; 109 110 if (cgid_len) 111 memcpy((void *)t2 + sizeof(*t2), &cgid, cgid_len); 112 if (pdu_len) 113 memcpy((void *)t2 + sizeof(*t2) + cgid_len, pdu_data, pdu_len); 114} 115 116static void relay_blktrace_event1(struct blk_trace *bt, unsigned long sequence, 117 pid_t pid, int cpu, sector_t sector, int bytes, 118 u64 what, int error, u64 cgid, 119 ssize_t cgid_len, void *pdu_data, int pdu_len) 120{ 121 struct blk_io_trace *t; 122 size_t trace_len = sizeof(*t) + pdu_len + cgid_len; 123 124 t = relay_reserve(bt->rchan, trace_len); 125 if (!t) 126 return; 127 128 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION; 129 t->sequence = sequence; 130 t->time = ktime_to_ns(ktime_get()); 131 132 record_blktrace_event(t, pid, cpu, sector, bytes, what, bt->dev, error, 133 cgid, cgid_len, pdu_data, pdu_len); 134} 135 136static void relay_blktrace_event2(struct blk_trace *bt, unsigned long sequence, 137 pid_t pid, int cpu, sector_t sector, 138 int bytes, u64 what, int error, u64 cgid, 139 ssize_t cgid_len, void *pdu_data, int pdu_len) 140{ 141 struct blk_io_trace2 *t; 142 size_t trace_len = sizeof(struct blk_io_trace2) + pdu_len + cgid_len; 143 144 t = relay_reserve(bt->rchan, trace_len); 145 if (!t) 146 return; 147 148 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE2_VERSION; 149 t->sequence = sequence; 150 t->time = ktime_to_ns(ktime_get()); 151 152 record_blktrace_event2(t, pid, cpu, sector, bytes, what, bt->dev, error, 153 cgid, cgid_len, pdu_data, pdu_len); 154} 155 156static void relay_blktrace_event(struct blk_trace *bt, unsigned long sequence, 157 pid_t pid, int cpu, sector_t sector, int bytes, 158 u64 what, int error, u64 cgid, 159 ssize_t cgid_len, void *pdu_data, int pdu_len) 160{ 161 if (bt->version == 2) 162 return relay_blktrace_event2(bt, sequence, pid, cpu, sector, 163 bytes, what, error, cgid, cgid_len, 164 pdu_data, pdu_len); 165 return relay_blktrace_event1(bt, sequence, pid, cpu, sector, bytes, 166 what, error, cgid, cgid_len, pdu_data, 167 pdu_len); 168} 169 170/* 171 * Send out a notify message. 172 */ 173static void trace_note(struct blk_trace *bt, pid_t pid, u64 action, 174 const void *data, size_t len, u64 cgid) 175{ 176 struct ring_buffer_event *event = NULL; 177 struct trace_buffer *buffer = NULL; 178 unsigned int trace_ctx = 0; 179 int cpu = smp_processor_id(); 180 bool blk_tracer = blk_tracer_enabled; 181 ssize_t cgid_len = cgid ? sizeof(cgid) : 0; 182 183 action = lower_32_bits(action | (cgid ? __BLK_TN_CGROUP : 0)); 184 if (blk_tracer) { 185 struct blk_io_trace2 *t; 186 size_t trace_len = sizeof(*t) + cgid_len + len; 187 188 buffer = blk_tr->array_buffer.buffer; 189 trace_ctx = tracing_gen_ctx_flags(0); 190 event = trace_buffer_lock_reserve(buffer, TRACE_BLK, 191 trace_len, trace_ctx); 192 if (!event) 193 return; 194 t = ring_buffer_event_data(event); 195 record_blktrace_event2(t, pid, cpu, 0, 0, 196 action, bt->dev, 0, cgid, cgid_len, 197 (void *)data, len); 198 trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx); 199 return; 200 } 201 202 if (!bt->rchan) 203 return; 204 205 relay_blktrace_event(bt, 0, pid, cpu, 0, 0, action, 0, cgid, 206 cgid_len, (void *)data, len); 207} 208 209/* 210 * Send out a notify for this process, if we haven't done so since a trace 211 * started 212 */ 213static void trace_note_tsk(struct task_struct *tsk) 214{ 215 unsigned long flags; 216 struct blk_trace *bt; 217 218 tsk->btrace_seq = blktrace_seq; 219 raw_spin_lock_irqsave(&running_trace_lock, flags); 220 list_for_each_entry(bt, &running_trace_list, running_list) { 221 trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, 222 sizeof(tsk->comm), 0); 223 } 224 raw_spin_unlock_irqrestore(&running_trace_lock, flags); 225} 226 227static void trace_note_time(struct blk_trace *bt) 228{ 229 struct timespec64 now; 230 unsigned long flags; 231 u32 words[2]; 232 233 /* need to check user space to see if this breaks in y2038 or y2106 */ 234 ktime_get_real_ts64(&now); 235 words[0] = (u32)now.tv_sec; 236 words[1] = now.tv_nsec; 237 238 local_irq_save(flags); 239 trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words), 0); 240 local_irq_restore(flags); 241} 242 243void __blk_trace_note_message(struct blk_trace *bt, 244 struct cgroup_subsys_state *css, const char *fmt, ...) 245{ 246 int n; 247 va_list args; 248 unsigned long flags; 249 char *buf; 250 u64 cgid = 0; 251 252 if (unlikely(bt->trace_state != Blktrace_running && 253 !blk_tracer_enabled)) 254 return; 255 256 /* 257 * If the BLK_TC_NOTIFY action mask isn't set, don't send any note 258 * message to the trace. 259 */ 260 if (!(bt->act_mask & BLK_TC_NOTIFY)) 261 return; 262 263 local_irq_save(flags); 264 buf = this_cpu_ptr(bt->msg_data); 265 va_start(args, fmt); 266 n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args); 267 va_end(args); 268 269#ifdef CONFIG_BLK_CGROUP 270 if (css && (blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) 271 cgid = cgroup_id(css->cgroup); 272 else 273 cgid = 1; 274#endif 275 trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, cgid); 276 local_irq_restore(flags); 277} 278EXPORT_SYMBOL_GPL(__blk_trace_note_message); 279 280static int act_log_check(struct blk_trace *bt, u64 what, sector_t sector, 281 pid_t pid) 282{ 283 if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0) 284 return 1; 285 if (sector && (sector < bt->start_lba || sector > bt->end_lba)) 286 return 1; 287 if (bt->pid && pid != bt->pid) 288 return 1; 289 290 return 0; 291} 292 293/* 294 * Data direction bit lookup 295 */ 296static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ), 297 BLK_TC_ACT(BLK_TC_WRITE) }; 298 299#define BLK_TC_RAHEAD BLK_TC_AHEAD 300#define BLK_TC_PREFLUSH BLK_TC_FLUSH 301 302/* The ilog2() calls fall out because they're constant */ 303#define MASK_TC_BIT(rw, __name) ((__force u32)(rw & REQ_ ## __name) << \ 304 (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name)) 305 306/* 307 * The worker for the various blk_add_trace*() types. Fills out a 308 * blk_io_trace structure and places it in a per-cpu subbuffer. 309 */ 310static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, 311 const blk_opf_t opf, u64 what, int error, 312 int pdu_len, void *pdu_data, u64 cgid) 313{ 314 struct task_struct *tsk = current; 315 struct ring_buffer_event *event = NULL; 316 struct trace_buffer *buffer = NULL; 317 unsigned long flags = 0; 318 unsigned long *sequence; 319 unsigned int trace_ctx = 0; 320 pid_t pid; 321 int cpu; 322 bool blk_tracer = blk_tracer_enabled; 323 ssize_t cgid_len = cgid ? sizeof(cgid) : 0; 324 const enum req_op op = opf & REQ_OP_MASK; 325 size_t trace_len; 326 327 if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer)) 328 return; 329 330 what |= ddir_act[op_is_write(op) ? WRITE : READ]; 331 what |= MASK_TC_BIT(opf, SYNC); 332 what |= MASK_TC_BIT(opf, RAHEAD); 333 what |= MASK_TC_BIT(opf, META); 334 what |= MASK_TC_BIT(opf, PREFLUSH); 335 what |= MASK_TC_BIT(opf, FUA); 336 337 switch (op) { 338 case REQ_OP_DISCARD: 339 case REQ_OP_SECURE_ERASE: 340 what |= BLK_TC_ACT(BLK_TC_DISCARD); 341 break; 342 case REQ_OP_FLUSH: 343 what |= BLK_TC_ACT(BLK_TC_FLUSH); 344 break; 345 case REQ_OP_ZONE_APPEND: 346 what |= BLK_TC_ACT(BLK_TC_ZONE_APPEND); 347 break; 348 case REQ_OP_ZONE_RESET: 349 what |= BLK_TC_ACT(BLK_TC_ZONE_RESET); 350 break; 351 case REQ_OP_ZONE_RESET_ALL: 352 what |= BLK_TC_ACT(BLK_TC_ZONE_RESET_ALL); 353 break; 354 case REQ_OP_ZONE_FINISH: 355 what |= BLK_TC_ACT(BLK_TC_ZONE_FINISH); 356 break; 357 case REQ_OP_ZONE_OPEN: 358 what |= BLK_TC_ACT(BLK_TC_ZONE_OPEN); 359 break; 360 case REQ_OP_ZONE_CLOSE: 361 what |= BLK_TC_ACT(BLK_TC_ZONE_CLOSE); 362 break; 363 case REQ_OP_WRITE_ZEROES: 364 what |= BLK_TC_ACT(BLK_TC_WRITE_ZEROES); 365 break; 366 default: 367 break; 368 } 369 370 /* Drop trace events for zone operations with blktrace v1 */ 371 if (bt->version == 1 && (what >> BLK_TC_SHIFT) > BLK_TC_END_V1) { 372 pr_debug_ratelimited("blktrace v1 cannot trace zone operation 0x%llx\n", 373 (unsigned long long)what); 374 return; 375 } 376 377 if (cgid) 378 what |= __BLK_TA_CGROUP; 379 380 pid = tsk->pid; 381 if (act_log_check(bt, what, sector, pid)) 382 return; 383 cpu = raw_smp_processor_id(); 384 385 if (blk_tracer) { 386 buffer = blk_tr->array_buffer.buffer; 387 trace_ctx = tracing_gen_ctx_flags(0); 388 switch (bt->version) { 389 case 1: 390 trace_len = sizeof(struct blk_io_trace); 391 break; 392 case 2: 393 default: 394 /* 395 * ftrace always uses v2 (blk_io_trace2) format. 396 * 397 * For sysfs-enabled tracing path (enabled via 398 * /sys/block/DEV/trace/enable), blk_trace_setup_queue() 399 * never initializes bt->version, leaving it 0 from 400 * kzalloc(). We must handle version==0 safely here. 401 * 402 * Fall through to default to ensure we never hit the 403 * old bug where default set trace_len=0, causing 404 * buffer underflow and memory corruption. 405 * 406 * Always use v2 format for ftrace and normalize 407 * bt->version to 2 when uninitialized. 408 */ 409 trace_len = sizeof(struct blk_io_trace2); 410 if (bt->version == 0) 411 bt->version = 2; 412 break; 413 } 414 trace_len += pdu_len + cgid_len; 415 event = trace_buffer_lock_reserve(buffer, TRACE_BLK, 416 trace_len, trace_ctx); 417 if (!event) 418 return; 419 420 tracing_record_cmdline(current); 421 switch (bt->version) { 422 case 1: 423 record_blktrace_event(ring_buffer_event_data(event), 424 pid, cpu, sector, bytes, 425 what, bt->dev, error, cgid, cgid_len, 426 pdu_data, pdu_len); 427 break; 428 case 2: 429 default: 430 /* 431 * Use v2 recording function (record_blktrace_event2) 432 * which writes blk_io_trace2 structure with correct 433 * field layout: 434 * - 32-bit pid at offset 28 435 * - 64-bit action at offset 32 436 * 437 * Fall through to default handles version==0 case 438 * (from sysfs path), ensuring we always use correct 439 * v2 recording function to match the v2 buffer 440 * allocated above. 441 */ 442 record_blktrace_event2(ring_buffer_event_data(event), 443 pid, cpu, sector, bytes, 444 what, bt->dev, error, cgid, cgid_len, 445 pdu_data, pdu_len); 446 break; 447 } 448 449 trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx); 450 return; 451 } 452 453 if (unlikely(tsk->btrace_seq != blktrace_seq)) 454 trace_note_tsk(tsk); 455 456 /* 457 * A word about the locking here - we disable interrupts to reserve 458 * some space in the relay per-cpu buffer, to prevent an irq 459 * from coming in and stepping on our toes. 460 */ 461 local_irq_save(flags); 462 sequence = per_cpu_ptr(bt->sequence, cpu); 463 (*sequence)++; 464 relay_blktrace_event(bt, *sequence, pid, cpu, sector, bytes, 465 what, error, cgid, cgid_len, pdu_data, pdu_len); 466 local_irq_restore(flags); 467} 468 469static void blk_trace_free(struct request_queue *q, struct blk_trace *bt) 470{ 471 relay_close(bt->rchan); 472 473 /* 474 * If 'bt->dir' is not set, then both 'dropped' and 'msg' are created 475 * under 'q->debugfs_dir', thus lookup and remove them. 476 */ 477 if (!bt->dir) { 478 debugfs_lookup_and_remove("dropped", q->debugfs_dir); 479 debugfs_lookup_and_remove("msg", q->debugfs_dir); 480 } else { 481 debugfs_remove(bt->dir); 482 } 483 free_percpu(bt->sequence); 484 free_percpu(bt->msg_data); 485 kfree(bt); 486} 487 488static void get_probe_ref(void) 489{ 490 mutex_lock(&blk_probe_mutex); 491 if (++blk_probes_ref == 1) 492 blk_register_tracepoints(); 493 mutex_unlock(&blk_probe_mutex); 494} 495 496static void put_probe_ref(void) 497{ 498 mutex_lock(&blk_probe_mutex); 499 if (!--blk_probes_ref) 500 blk_unregister_tracepoints(); 501 mutex_unlock(&blk_probe_mutex); 502} 503 504static int blk_trace_start(struct blk_trace *bt) 505{ 506 if (bt->trace_state != Blktrace_setup && 507 bt->trace_state != Blktrace_stopped) 508 return -EINVAL; 509 510 blktrace_seq++; 511 smp_mb(); 512 bt->trace_state = Blktrace_running; 513 raw_spin_lock_irq(&running_trace_lock); 514 list_add(&bt->running_list, &running_trace_list); 515 raw_spin_unlock_irq(&running_trace_lock); 516 trace_note_time(bt); 517 518 return 0; 519} 520 521static int blk_trace_stop(struct blk_trace *bt) 522{ 523 if (bt->trace_state != Blktrace_running) 524 return -EINVAL; 525 526 bt->trace_state = Blktrace_stopped; 527 raw_spin_lock_irq(&running_trace_lock); 528 list_del_init(&bt->running_list); 529 raw_spin_unlock_irq(&running_trace_lock); 530 relay_flush(bt->rchan); 531 532 return 0; 533} 534 535static void blk_trace_cleanup(struct request_queue *q, struct blk_trace *bt) 536{ 537 blk_trace_stop(bt); 538 synchronize_rcu(); 539 blk_trace_free(q, bt); 540 put_probe_ref(); 541} 542 543static int __blk_trace_remove(struct request_queue *q) 544{ 545 struct blk_trace *bt; 546 547 bt = rcu_replace_pointer(q->blk_trace, NULL, 548 lockdep_is_held(&q->debugfs_mutex)); 549 if (!bt) 550 return -EINVAL; 551 552 blk_trace_cleanup(q, bt); 553 554 return 0; 555} 556 557int blk_trace_remove(struct request_queue *q) 558{ 559 int ret; 560 561 mutex_lock(&q->debugfs_mutex); 562 ret = __blk_trace_remove(q); 563 mutex_unlock(&q->debugfs_mutex); 564 565 return ret; 566} 567EXPORT_SYMBOL_GPL(blk_trace_remove); 568 569static ssize_t blk_dropped_read(struct file *filp, char __user *buffer, 570 size_t count, loff_t *ppos) 571{ 572 struct blk_trace *bt = filp->private_data; 573 size_t dropped = relay_stats(bt->rchan, RELAY_STATS_BUF_FULL); 574 char buf[16]; 575 576 snprintf(buf, sizeof(buf), "%zu\n", dropped); 577 578 return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf)); 579} 580 581static const struct file_operations blk_dropped_fops = { 582 .owner = THIS_MODULE, 583 .open = simple_open, 584 .read = blk_dropped_read, 585 .llseek = default_llseek, 586}; 587 588static ssize_t blk_msg_write(struct file *filp, const char __user *buffer, 589 size_t count, loff_t *ppos) 590{ 591 char *msg; 592 struct blk_trace *bt; 593 594 if (count >= BLK_TN_MAX_MSG) 595 return -EINVAL; 596 597 msg = memdup_user_nul(buffer, count); 598 if (IS_ERR(msg)) 599 return PTR_ERR(msg); 600 601 bt = filp->private_data; 602 __blk_trace_note_message(bt, NULL, "%s", msg); 603 kfree(msg); 604 605 return count; 606} 607 608static const struct file_operations blk_msg_fops = { 609 .owner = THIS_MODULE, 610 .open = simple_open, 611 .write = blk_msg_write, 612 .llseek = noop_llseek, 613}; 614 615static int blk_remove_buf_file_callback(struct dentry *dentry) 616{ 617 debugfs_remove(dentry); 618 619 return 0; 620} 621 622static struct dentry *blk_create_buf_file_callback(const char *filename, 623 struct dentry *parent, 624 umode_t mode, 625 struct rchan_buf *buf, 626 int *is_global) 627{ 628 return debugfs_create_file(filename, mode, parent, buf, 629 &relay_file_operations); 630} 631 632static const struct rchan_callbacks blk_relay_callbacks = { 633 .create_buf_file = blk_create_buf_file_callback, 634 .remove_buf_file = blk_remove_buf_file_callback, 635}; 636 637static void blk_trace_setup_lba(struct blk_trace *bt, 638 struct block_device *bdev) 639{ 640 if (bdev) { 641 bt->start_lba = bdev->bd_start_sect; 642 bt->end_lba = bdev->bd_start_sect + bdev_nr_sectors(bdev); 643 } else { 644 bt->start_lba = 0; 645 bt->end_lba = -1ULL; 646 } 647} 648 649/* 650 * Setup everything required to start tracing 651 */ 652static struct blk_trace *blk_trace_setup_prepare(struct request_queue *q, 653 char *name, dev_t dev, 654 u32 buf_size, u32 buf_nr, 655 struct block_device *bdev) 656{ 657 struct blk_trace *bt = NULL; 658 struct dentry *dir = NULL; 659 int ret; 660 661 lockdep_assert_held(&q->debugfs_mutex); 662 663 /* 664 * bdev can be NULL, as with scsi-generic, this is a helpful as 665 * we can be. 666 */ 667 if (rcu_dereference_protected(q->blk_trace, 668 lockdep_is_held(&q->debugfs_mutex))) { 669 pr_warn("Concurrent blktraces are not allowed on %s\n", name); 670 return ERR_PTR(-EBUSY); 671 } 672 673 bt = kzalloc(sizeof(*bt), GFP_KERNEL); 674 if (!bt) 675 return ERR_PTR(-ENOMEM); 676 677 ret = -ENOMEM; 678 bt->sequence = alloc_percpu(unsigned long); 679 if (!bt->sequence) 680 goto err; 681 682 bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char)); 683 if (!bt->msg_data) 684 goto err; 685 686 /* 687 * When tracing the whole disk reuse the existing debugfs directory 688 * created by the block layer on init. For partitions block devices, 689 * and scsi-generic block devices we create a temporary new debugfs 690 * directory that will be removed once the trace ends. 691 */ 692 if (bdev && !bdev_is_partition(bdev)) 693 dir = q->debugfs_dir; 694 else 695 bt->dir = dir = debugfs_create_dir(name, blk_debugfs_root); 696 697 /* 698 * As blktrace relies on debugfs for its interface the debugfs directory 699 * is required, contrary to the usual mantra of not checking for debugfs 700 * files or directories. 701 */ 702 if (IS_ERR_OR_NULL(dir)) { 703 pr_warn("debugfs_dir not present for %s so skipping\n", name); 704 ret = -ENOENT; 705 goto err; 706 } 707 708 bt->dev = dev; 709 INIT_LIST_HEAD(&bt->running_list); 710 711 ret = -EIO; 712 debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops); 713 debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops); 714 715 bt->rchan = relay_open("trace", dir, buf_size, buf_nr, 716 &blk_relay_callbacks, bt); 717 if (!bt->rchan) 718 goto err; 719 720 blk_trace_setup_lba(bt, bdev); 721 722 return bt; 723 724err: 725 blk_trace_free(q, bt); 726 727 return ERR_PTR(ret); 728} 729 730static void blk_trace_setup_finalize(struct request_queue *q, 731 char *name, int version, 732 struct blk_trace *bt, 733 struct blk_user_trace_setup2 *buts) 734 735{ 736 strscpy_pad(buts->name, name, BLKTRACE_BDEV_SIZE2); 737 738 /* 739 * some device names have larger paths - convert the slashes 740 * to underscores for this to work as expected 741 */ 742 strreplace(buts->name, '/', '_'); 743 744 bt->version = version; 745 bt->act_mask = buts->act_mask; 746 if (!bt->act_mask) 747 bt->act_mask = (u16) -1; 748 749 /* overwrite with user settings */ 750 if (buts->start_lba) 751 bt->start_lba = buts->start_lba; 752 if (buts->end_lba) 753 bt->end_lba = buts->end_lba; 754 755 bt->pid = buts->pid; 756 bt->trace_state = Blktrace_setup; 757 758 rcu_assign_pointer(q->blk_trace, bt); 759 get_probe_ref(); 760} 761 762int blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 763 struct block_device *bdev, 764 char __user *arg) 765{ 766 struct blk_user_trace_setup2 buts2; 767 struct blk_user_trace_setup buts; 768 struct blk_trace *bt; 769 int ret; 770 771 ret = copy_from_user(&buts, arg, sizeof(buts)); 772 if (ret) 773 return -EFAULT; 774 775 if (!buts.buf_size || !buts.buf_nr) 776 return -EINVAL; 777 778 buts2 = (struct blk_user_trace_setup2) { 779 .act_mask = buts.act_mask, 780 .buf_size = buts.buf_size, 781 .buf_nr = buts.buf_nr, 782 .start_lba = buts.start_lba, 783 .end_lba = buts.end_lba, 784 .pid = buts.pid, 785 }; 786 787 mutex_lock(&q->debugfs_mutex); 788 bt = blk_trace_setup_prepare(q, name, dev, buts.buf_size, buts.buf_nr, 789 bdev); 790 if (IS_ERR(bt)) { 791 mutex_unlock(&q->debugfs_mutex); 792 return PTR_ERR(bt); 793 } 794 blk_trace_setup_finalize(q, name, 1, bt, &buts2); 795 strscpy(buts.name, buts2.name, BLKTRACE_BDEV_SIZE); 796 mutex_unlock(&q->debugfs_mutex); 797 798 if (copy_to_user(arg, &buts, sizeof(buts))) { 799 blk_trace_remove(q); 800 return -EFAULT; 801 } 802 return 0; 803} 804EXPORT_SYMBOL_GPL(blk_trace_setup); 805 806static int blk_trace_setup2(struct request_queue *q, char *name, dev_t dev, 807 struct block_device *bdev, char __user *arg) 808{ 809 struct blk_user_trace_setup2 buts2; 810 struct blk_trace *bt; 811 812 if (copy_from_user(&buts2, arg, sizeof(buts2))) 813 return -EFAULT; 814 815 if (!buts2.buf_size || !buts2.buf_nr) 816 return -EINVAL; 817 818 if (buts2.flags != 0) 819 return -EINVAL; 820 821 mutex_lock(&q->debugfs_mutex); 822 bt = blk_trace_setup_prepare(q, name, dev, buts2.buf_size, buts2.buf_nr, 823 bdev); 824 if (IS_ERR(bt)) { 825 mutex_unlock(&q->debugfs_mutex); 826 return PTR_ERR(bt); 827 } 828 blk_trace_setup_finalize(q, name, 2, bt, &buts2); 829 mutex_unlock(&q->debugfs_mutex); 830 831 if (copy_to_user(arg, &buts2, sizeof(buts2))) { 832 blk_trace_remove(q); 833 return -EFAULT; 834 } 835 return 0; 836} 837 838#if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) 839static int compat_blk_trace_setup(struct request_queue *q, char *name, 840 dev_t dev, struct block_device *bdev, 841 char __user *arg) 842{ 843 struct blk_user_trace_setup2 buts2; 844 struct compat_blk_user_trace_setup cbuts; 845 struct blk_trace *bt; 846 847 if (copy_from_user(&cbuts, arg, sizeof(cbuts))) 848 return -EFAULT; 849 850 if (!cbuts.buf_size || !cbuts.buf_nr) 851 return -EINVAL; 852 853 buts2 = (struct blk_user_trace_setup2) { 854 .act_mask = cbuts.act_mask, 855 .buf_size = cbuts.buf_size, 856 .buf_nr = cbuts.buf_nr, 857 .start_lba = cbuts.start_lba, 858 .end_lba = cbuts.end_lba, 859 .pid = cbuts.pid, 860 }; 861 862 mutex_lock(&q->debugfs_mutex); 863 bt = blk_trace_setup_prepare(q, name, dev, buts2.buf_size, buts2.buf_nr, 864 bdev); 865 if (IS_ERR(bt)) { 866 mutex_unlock(&q->debugfs_mutex); 867 return PTR_ERR(bt); 868 } 869 blk_trace_setup_finalize(q, name, 1, bt, &buts2); 870 mutex_unlock(&q->debugfs_mutex); 871 872 if (copy_to_user(arg, &buts2.name, ARRAY_SIZE(buts2.name))) { 873 blk_trace_remove(q); 874 return -EFAULT; 875 } 876 877 return 0; 878} 879#endif 880 881static int __blk_trace_startstop(struct request_queue *q, int start) 882{ 883 struct blk_trace *bt; 884 885 bt = rcu_dereference_protected(q->blk_trace, 886 lockdep_is_held(&q->debugfs_mutex)); 887 if (bt == NULL) 888 return -EINVAL; 889 890 if (start) 891 return blk_trace_start(bt); 892 else 893 return blk_trace_stop(bt); 894} 895 896int blk_trace_startstop(struct request_queue *q, int start) 897{ 898 int ret; 899 900 mutex_lock(&q->debugfs_mutex); 901 ret = __blk_trace_startstop(q, start); 902 mutex_unlock(&q->debugfs_mutex); 903 904 return ret; 905} 906EXPORT_SYMBOL_GPL(blk_trace_startstop); 907 908/* 909 * When reading or writing the blktrace sysfs files, the references to the 910 * opened sysfs or device files should prevent the underlying block device 911 * from being removed. So no further delete protection is really needed. 912 */ 913 914/** 915 * blk_trace_ioctl - handle the ioctls associated with tracing 916 * @bdev: the block device 917 * @cmd: the ioctl cmd 918 * @arg: the argument data, if any 919 * 920 **/ 921int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) 922{ 923 struct request_queue *q = bdev_get_queue(bdev); 924 int ret, start = 0; 925 char b[BDEVNAME_SIZE]; 926 927 switch (cmd) { 928 case BLKTRACESETUP2: 929 snprintf(b, sizeof(b), "%pg", bdev); 930 ret = blk_trace_setup2(q, b, bdev->bd_dev, bdev, arg); 931 break; 932 case BLKTRACESETUP: 933 snprintf(b, sizeof(b), "%pg", bdev); 934 ret = blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); 935 break; 936#if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) 937 case BLKTRACESETUP32: 938 snprintf(b, sizeof(b), "%pg", bdev); 939 ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); 940 break; 941#endif 942 case BLKTRACESTART: 943 start = 1; 944 fallthrough; 945 case BLKTRACESTOP: 946 ret = blk_trace_startstop(q, start); 947 break; 948 case BLKTRACETEARDOWN: 949 ret = blk_trace_remove(q); 950 break; 951 default: 952 ret = -ENOTTY; 953 break; 954 } 955 return ret; 956} 957 958/** 959 * blk_trace_shutdown - stop and cleanup trace structures 960 * @q: the request queue associated with the device 961 * 962 **/ 963void blk_trace_shutdown(struct request_queue *q) 964{ 965 if (rcu_dereference_protected(q->blk_trace, 966 lockdep_is_held(&q->debugfs_mutex))) 967 __blk_trace_remove(q); 968} 969 970#ifdef CONFIG_BLK_CGROUP 971static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) 972{ 973 struct cgroup_subsys_state *blkcg_css; 974 struct blk_trace *bt; 975 976 /* We don't use the 'bt' value here except as an optimization... */ 977 bt = rcu_dereference_protected(q->blk_trace, 1); 978 if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) 979 return 0; 980 981 blkcg_css = bio_blkcg_css(bio); 982 if (!blkcg_css) 983 return 0; 984 return cgroup_id(blkcg_css->cgroup); 985} 986#else 987static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) 988{ 989 return 0; 990} 991#endif 992 993static u64 994blk_trace_request_get_cgid(struct request *rq) 995{ 996 if (!rq->bio) 997 return 0; 998 /* Use the first bio */ 999 return blk_trace_bio_get_cgid(rq->q, rq->bio); 1000} 1001 1002/* 1003 * blktrace probes 1004 */ 1005 1006/** 1007 * blk_add_trace_rq - Add a trace for a request oriented action 1008 * @rq: the source request 1009 * @error: return status to log 1010 * @nr_bytes: number of completed bytes 1011 * @what: the action 1012 * @cgid: the cgroup info 1013 * 1014 * Description: 1015 * Records an action against a request. Will log the bio offset + size. 1016 * 1017 **/ 1018static void blk_add_trace_rq(struct request *rq, blk_status_t error, 1019 unsigned int nr_bytes, u64 what, u64 cgid) 1020{ 1021 struct blk_trace *bt; 1022 1023 rcu_read_lock(); 1024 bt = rcu_dereference(rq->q->blk_trace); 1025 if (likely(!bt)) { 1026 rcu_read_unlock(); 1027 return; 1028 } 1029 1030 if (blk_rq_is_passthrough(rq)) 1031 what |= BLK_TC_ACT(BLK_TC_PC); 1032 else 1033 what |= BLK_TC_ACT(BLK_TC_FS); 1034 1035 __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, rq->cmd_flags, 1036 what, blk_status_to_errno(error), 0, NULL, cgid); 1037 rcu_read_unlock(); 1038} 1039 1040static void blk_add_trace_rq_insert(void *ignore, struct request *rq) 1041{ 1042 blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_INSERT, 1043 blk_trace_request_get_cgid(rq)); 1044} 1045 1046static void blk_add_trace_rq_issue(void *ignore, struct request *rq) 1047{ 1048 blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_ISSUE, 1049 blk_trace_request_get_cgid(rq)); 1050} 1051 1052static void blk_add_trace_rq_merge(void *ignore, struct request *rq) 1053{ 1054 blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_BACKMERGE, 1055 blk_trace_request_get_cgid(rq)); 1056} 1057 1058static void blk_add_trace_rq_requeue(void *ignore, struct request *rq) 1059{ 1060 blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_REQUEUE, 1061 blk_trace_request_get_cgid(rq)); 1062} 1063 1064static void blk_add_trace_rq_complete(void *ignore, struct request *rq, 1065 blk_status_t error, unsigned int nr_bytes) 1066{ 1067 blk_add_trace_rq(rq, error, nr_bytes, BLK_TA_COMPLETE, 1068 blk_trace_request_get_cgid(rq)); 1069} 1070 1071static void blk_add_trace_zone_update_request(void *ignore, struct request *rq) 1072{ 1073 struct blk_trace *bt; 1074 1075 rcu_read_lock(); 1076 bt = rcu_dereference(rq->q->blk_trace); 1077 if (likely(!bt) || bt->version < 2) { 1078 rcu_read_unlock(); 1079 return; 1080 } 1081 rcu_read_unlock(); 1082 1083 blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_ZONE_APPEND, 1084 blk_trace_request_get_cgid(rq)); 1085} 1086 1087/** 1088 * blk_add_trace_bio - Add a trace for a bio oriented action 1089 * @q: queue the io is for 1090 * @bio: the source bio 1091 * @what: the action 1092 * @error: error, if any 1093 * 1094 * Description: 1095 * Records an action against a bio. Will log the bio offset + size. 1096 * 1097 **/ 1098static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, 1099 u64 what, int error) 1100{ 1101 struct blk_trace *bt; 1102 1103 rcu_read_lock(); 1104 bt = rcu_dereference(q->blk_trace); 1105 if (likely(!bt)) { 1106 rcu_read_unlock(); 1107 return; 1108 } 1109 1110 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 1111 bio->bi_opf, what, error, 0, NULL, 1112 blk_trace_bio_get_cgid(q, bio)); 1113 rcu_read_unlock(); 1114} 1115 1116static void blk_add_trace_bio_complete(void *ignore, 1117 struct request_queue *q, struct bio *bio) 1118{ 1119 blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, 1120 blk_status_to_errno(bio->bi_status)); 1121} 1122 1123static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio) 1124{ 1125 blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BACKMERGE, 1126 0); 1127} 1128 1129static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio) 1130{ 1131 blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_FRONTMERGE, 1132 0); 1133} 1134 1135static void blk_add_trace_bio_queue(void *ignore, struct bio *bio) 1136{ 1137 blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_QUEUE, 0); 1138} 1139 1140static void blk_add_trace_getrq(void *ignore, struct bio *bio) 1141{ 1142 blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_GETRQ, 0); 1143} 1144 1145static void blk_add_trace_plug(void *ignore, struct request_queue *q) 1146{ 1147 struct blk_trace *bt; 1148 1149 rcu_read_lock(); 1150 bt = rcu_dereference(q->blk_trace); 1151 if (bt) 1152 __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0); 1153 rcu_read_unlock(); 1154} 1155 1156static void blk_add_trace_unplug(void *ignore, struct request_queue *q, 1157 unsigned int depth, bool explicit) 1158{ 1159 struct blk_trace *bt; 1160 1161 rcu_read_lock(); 1162 bt = rcu_dereference(q->blk_trace); 1163 if (bt) { 1164 __be64 rpdu = cpu_to_be64(depth); 1165 u64 what; 1166 1167 if (explicit) 1168 what = BLK_TA_UNPLUG_IO; 1169 else 1170 what = BLK_TA_UNPLUG_TIMER; 1171 1172 __blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0); 1173 } 1174 rcu_read_unlock(); 1175} 1176 1177static void blk_add_trace_zone_plug(void *ignore, struct request_queue *q, 1178 unsigned int zno, sector_t sector, 1179 unsigned int sectors) 1180{ 1181 struct blk_trace *bt; 1182 1183 rcu_read_lock(); 1184 bt = rcu_dereference(q->blk_trace); 1185 if (bt && bt->version >= 2) 1186 __blk_add_trace(bt, sector, sectors << SECTOR_SHIFT, 0, 1187 BLK_TA_ZONE_PLUG, 0, 0, NULL, 0); 1188 rcu_read_unlock(); 1189 1190 return; 1191} 1192 1193static void blk_add_trace_zone_unplug(void *ignore, struct request_queue *q, 1194 unsigned int zno, sector_t sector, 1195 unsigned int sectors) 1196{ 1197 struct blk_trace *bt; 1198 1199 rcu_read_lock(); 1200 bt = rcu_dereference(q->blk_trace); 1201 if (bt && bt->version >= 2) 1202 __blk_add_trace(bt, sector, sectors << SECTOR_SHIFT, 0, 1203 BLK_TA_ZONE_UNPLUG, 0, 0, NULL, 0); 1204 rcu_read_unlock(); 1205 return; 1206} 1207 1208static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) 1209{ 1210 struct request_queue *q = bio->bi_bdev->bd_disk->queue; 1211 struct blk_trace *bt; 1212 1213 rcu_read_lock(); 1214 bt = rcu_dereference(q->blk_trace); 1215 if (bt) { 1216 __be64 rpdu = cpu_to_be64(pdu); 1217 1218 __blk_add_trace(bt, bio->bi_iter.bi_sector, 1219 bio->bi_iter.bi_size, bio->bi_opf, BLK_TA_SPLIT, 1220 blk_status_to_errno(bio->bi_status), 1221 sizeof(rpdu), &rpdu, 1222 blk_trace_bio_get_cgid(q, bio)); 1223 } 1224 rcu_read_unlock(); 1225} 1226 1227/** 1228 * blk_add_trace_bio_remap - Add a trace for a bio-remap operation 1229 * @ignore: trace callback data parameter (not used) 1230 * @bio: the source bio 1231 * @dev: source device 1232 * @from: source sector 1233 * 1234 * Called after a bio is remapped to a different device and/or sector. 1235 **/ 1236static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev, 1237 sector_t from) 1238{ 1239 struct request_queue *q = bio->bi_bdev->bd_disk->queue; 1240 struct blk_trace *bt; 1241 struct blk_io_trace_remap r; 1242 1243 rcu_read_lock(); 1244 bt = rcu_dereference(q->blk_trace); 1245 if (likely(!bt)) { 1246 rcu_read_unlock(); 1247 return; 1248 } 1249 1250 r.device_from = cpu_to_be32(dev); 1251 r.device_to = cpu_to_be32(bio_dev(bio)); 1252 r.sector_from = cpu_to_be64(from); 1253 1254 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 1255 bio->bi_opf, BLK_TA_REMAP, 1256 blk_status_to_errno(bio->bi_status), 1257 sizeof(r), &r, blk_trace_bio_get_cgid(q, bio)); 1258 rcu_read_unlock(); 1259} 1260 1261/** 1262 * blk_add_trace_rq_remap - Add a trace for a request-remap operation 1263 * @ignore: trace callback data parameter (not used) 1264 * @rq: the source request 1265 * @dev: target device 1266 * @from: source sector 1267 * 1268 * Description: 1269 * Device mapper remaps request to other devices. 1270 * Add a trace for that action. 1271 * 1272 **/ 1273static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev, 1274 sector_t from) 1275{ 1276 struct blk_trace *bt; 1277 struct blk_io_trace_remap r; 1278 1279 rcu_read_lock(); 1280 bt = rcu_dereference(rq->q->blk_trace); 1281 if (likely(!bt)) { 1282 rcu_read_unlock(); 1283 return; 1284 } 1285 1286 r.device_from = cpu_to_be32(dev); 1287 r.device_to = cpu_to_be32(disk_devt(rq->q->disk)); 1288 r.sector_from = cpu_to_be64(from); 1289 1290 __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 1291 rq->cmd_flags, BLK_TA_REMAP, 0, 1292 sizeof(r), &r, blk_trace_request_get_cgid(rq)); 1293 rcu_read_unlock(); 1294} 1295 1296/** 1297 * blk_add_driver_data - Add binary message with driver-specific data 1298 * @rq: io request 1299 * @data: driver-specific data 1300 * @len: length of driver-specific data 1301 * 1302 * Description: 1303 * Some drivers might want to write driver-specific data per request. 1304 * 1305 **/ 1306void blk_add_driver_data(struct request *rq, void *data, size_t len) 1307{ 1308 struct blk_trace *bt; 1309 1310 rcu_read_lock(); 1311 bt = rcu_dereference(rq->q->blk_trace); 1312 if (likely(!bt)) { 1313 rcu_read_unlock(); 1314 return; 1315 } 1316 1317 __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 1318 BLK_TA_DRV_DATA, 0, len, data, 1319 blk_trace_request_get_cgid(rq)); 1320 rcu_read_unlock(); 1321} 1322EXPORT_SYMBOL_GPL(blk_add_driver_data); 1323 1324static void blk_register_tracepoints(void) 1325{ 1326 int ret; 1327 1328 ret = register_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); 1329 WARN_ON(ret); 1330 ret = register_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); 1331 WARN_ON(ret); 1332 ret = register_trace_block_rq_merge(blk_add_trace_rq_merge, NULL); 1333 WARN_ON(ret); 1334 ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); 1335 WARN_ON(ret); 1336 ret = register_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); 1337 WARN_ON(ret); 1338 ret = register_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); 1339 WARN_ON(ret); 1340 ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); 1341 WARN_ON(ret); 1342 ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); 1343 WARN_ON(ret); 1344 ret = register_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); 1345 WARN_ON(ret); 1346 ret = register_trace_block_getrq(blk_add_trace_getrq, NULL); 1347 WARN_ON(ret); 1348 ret = register_trace_blk_zone_append_update_request_bio( 1349 blk_add_trace_zone_update_request, NULL); 1350 WARN_ON(ret); 1351 ret = register_trace_disk_zone_wplug_add_bio(blk_add_trace_zone_plug, 1352 NULL); 1353 WARN_ON(ret); 1354 ret = register_trace_blk_zone_wplug_bio(blk_add_trace_zone_unplug, 1355 NULL); 1356 WARN_ON(ret); 1357 ret = register_trace_block_plug(blk_add_trace_plug, NULL); 1358 WARN_ON(ret); 1359 ret = register_trace_block_unplug(blk_add_trace_unplug, NULL); 1360 WARN_ON(ret); 1361 ret = register_trace_block_split(blk_add_trace_split, NULL); 1362 WARN_ON(ret); 1363 ret = register_trace_block_bio_remap(blk_add_trace_bio_remap, NULL); 1364 WARN_ON(ret); 1365 ret = register_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); 1366 WARN_ON(ret); 1367} 1368 1369static void blk_unregister_tracepoints(void) 1370{ 1371 unregister_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); 1372 unregister_trace_block_bio_remap(blk_add_trace_bio_remap, NULL); 1373 unregister_trace_block_split(blk_add_trace_split, NULL); 1374 unregister_trace_block_unplug(blk_add_trace_unplug, NULL); 1375 unregister_trace_block_plug(blk_add_trace_plug, NULL); 1376 unregister_trace_blk_zone_wplug_bio(blk_add_trace_zone_unplug, NULL); 1377 unregister_trace_disk_zone_wplug_add_bio(blk_add_trace_zone_plug, NULL); 1378 unregister_trace_blk_zone_append_update_request_bio( 1379 blk_add_trace_zone_update_request, NULL); 1380 unregister_trace_block_getrq(blk_add_trace_getrq, NULL); 1381 unregister_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); 1382 unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); 1383 unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); 1384 unregister_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); 1385 unregister_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); 1386 unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); 1387 unregister_trace_block_rq_merge(blk_add_trace_rq_merge, NULL); 1388 unregister_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); 1389 unregister_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); 1390 1391 tracepoint_synchronize_unregister(); 1392} 1393 1394/* 1395 * struct blk_io_tracer formatting routines 1396 */ 1397 1398static void fill_rwbs(char *rwbs, const struct blk_io_trace2 *t) 1399{ 1400 int i = 0; 1401 int tc = t->action >> BLK_TC_SHIFT; 1402 1403 if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) { 1404 rwbs[i++] = 'N'; 1405 goto out; 1406 } 1407 1408 if (tc & BLK_TC_FLUSH) 1409 rwbs[i++] = 'F'; 1410 1411 if (tc & BLK_TC_DISCARD) 1412 rwbs[i++] = 'D'; 1413 else if (tc & BLK_TC_WRITE_ZEROES) { 1414 rwbs[i++] = 'W'; 1415 rwbs[i++] = 'Z'; 1416 } else if (tc & BLK_TC_WRITE) 1417 rwbs[i++] = 'W'; 1418 else if (t->bytes) 1419 rwbs[i++] = 'R'; 1420 else 1421 rwbs[i++] = 'N'; 1422 1423 if (tc & BLK_TC_FUA) 1424 rwbs[i++] = 'F'; 1425 if (tc & BLK_TC_AHEAD) 1426 rwbs[i++] = 'A'; 1427 if (tc & BLK_TC_SYNC) 1428 rwbs[i++] = 'S'; 1429 if (tc & BLK_TC_META) 1430 rwbs[i++] = 'M'; 1431out: 1432 rwbs[i] = '\0'; 1433} 1434 1435static inline 1436const struct blk_io_trace2 *te_blk_io_trace(const struct trace_entry *ent) 1437{ 1438 return (const struct blk_io_trace2 *)ent; 1439} 1440 1441static inline const void *pdu_start(const struct trace_entry *ent, bool has_cg) 1442{ 1443 return (void *)(te_blk_io_trace(ent) + 1) + (has_cg ? sizeof(u64) : 0); 1444} 1445 1446static inline u64 t_cgid(const struct trace_entry *ent) 1447{ 1448 return *(u64 *)(te_blk_io_trace(ent) + 1); 1449} 1450 1451static inline int pdu_real_len(const struct trace_entry *ent, bool has_cg) 1452{ 1453 return te_blk_io_trace(ent)->pdu_len - (has_cg ? sizeof(u64) : 0); 1454} 1455 1456static inline u32 t_action(const struct trace_entry *ent) 1457{ 1458 return te_blk_io_trace(ent)->action; 1459} 1460 1461static inline u32 t_bytes(const struct trace_entry *ent) 1462{ 1463 return te_blk_io_trace(ent)->bytes; 1464} 1465 1466static inline u32 t_sec(const struct trace_entry *ent) 1467{ 1468 return te_blk_io_trace(ent)->bytes >> 9; 1469} 1470 1471static inline unsigned long long t_sector(const struct trace_entry *ent) 1472{ 1473 return te_blk_io_trace(ent)->sector; 1474} 1475 1476static inline __u16 t_error(const struct trace_entry *ent) 1477{ 1478 return te_blk_io_trace(ent)->error; 1479} 1480 1481static __u64 get_pdu_int(const struct trace_entry *ent, bool has_cg) 1482{ 1483 const __be64 *val = pdu_start(ent, has_cg); 1484 return be64_to_cpu(*val); 1485} 1486 1487typedef void (blk_log_action_t) (struct trace_iterator *iter, const char *act, 1488 bool has_cg); 1489 1490static void blk_log_action_classic(struct trace_iterator *iter, const char *act, 1491 bool has_cg) 1492{ 1493 char rwbs[RWBS_LEN]; 1494 unsigned long long ts = iter->ts; 1495 unsigned long nsec_rem = do_div(ts, NSEC_PER_SEC); 1496 unsigned secs = (unsigned long)ts; 1497 const struct blk_io_trace2 *t = te_blk_io_trace(iter->ent); 1498 1499 fill_rwbs(rwbs, t); 1500 1501 trace_seq_printf(&iter->seq, 1502 "%3d,%-3d %2d %5d.%09lu %5u %2s %3s ", 1503 MAJOR(t->device), MINOR(t->device), iter->cpu, 1504 secs, nsec_rem, iter->ent->pid, act, rwbs); 1505} 1506 1507static void blk_log_action(struct trace_iterator *iter, const char *act, 1508 bool has_cg) 1509{ 1510 char rwbs[RWBS_LEN]; 1511 const struct blk_io_trace2 *t = te_blk_io_trace(iter->ent); 1512 1513 fill_rwbs(rwbs, t); 1514 if (has_cg) { 1515 u64 id = t_cgid(iter->ent); 1516 1517 if (blk_tracer_flags.val & TRACE_BLK_OPT_CGNAME) { 1518 char blkcg_name_buf[NAME_MAX + 1] = "<...>"; 1519 1520 cgroup_path_from_kernfs_id(id, blkcg_name_buf, 1521 sizeof(blkcg_name_buf)); 1522 trace_seq_printf(&iter->seq, "%3d,%-3d %s %2s %3s ", 1523 MAJOR(t->device), MINOR(t->device), 1524 blkcg_name_buf, act, rwbs); 1525 } else { 1526 /* 1527 * The cgid portion used to be "INO,GEN". Userland 1528 * builds a FILEID_INO32_GEN fid out of them and 1529 * opens the cgroup using open_by_handle_at(2). 1530 * While 32bit ino setups are still the same, 64bit 1531 * ones now use the 64bit ino as the whole ID and 1532 * no longer use generation. 1533 * 1534 * Regardless of the content, always output 1535 * "LOW32,HIGH32" so that FILEID_INO32_GEN fid can 1536 * be mapped back to @id on both 64 and 32bit ino 1537 * setups. See __kernfs_fh_to_dentry(). 1538 */ 1539 trace_seq_printf(&iter->seq, 1540 "%3d,%-3d %llx,%-llx %2s %3s ", 1541 MAJOR(t->device), MINOR(t->device), 1542 id & U32_MAX, id >> 32, act, rwbs); 1543 } 1544 } else 1545 trace_seq_printf(&iter->seq, "%3d,%-3d %2s %3s ", 1546 MAJOR(t->device), MINOR(t->device), act, rwbs); 1547} 1548 1549static void blk_log_dump_pdu(struct trace_seq *s, 1550 const struct trace_entry *ent, bool has_cg) 1551{ 1552 const unsigned char *pdu_buf; 1553 int pdu_len; 1554 int i, end; 1555 1556 pdu_buf = pdu_start(ent, has_cg); 1557 pdu_len = pdu_real_len(ent, has_cg); 1558 1559 if (!pdu_len) 1560 return; 1561 1562 /* find the last zero that needs to be printed */ 1563 for (end = pdu_len - 1; end >= 0; end--) 1564 if (pdu_buf[end]) 1565 break; 1566 end++; 1567 1568 trace_seq_putc(s, '('); 1569 1570 for (i = 0; i < pdu_len; i++) { 1571 1572 trace_seq_printf(s, "%s%02x", 1573 i == 0 ? "" : " ", pdu_buf[i]); 1574 1575 /* 1576 * stop when the rest is just zeros and indicate so 1577 * with a ".." appended 1578 */ 1579 if (i == end && end != pdu_len - 1) { 1580 trace_seq_puts(s, " ..) "); 1581 return; 1582 } 1583 } 1584 1585 trace_seq_puts(s, ") "); 1586} 1587 1588static void blk_log_generic(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 1589{ 1590 char cmd[TASK_COMM_LEN]; 1591 1592 trace_find_cmdline(ent->pid, cmd); 1593 1594 if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) { 1595 trace_seq_printf(s, "%u ", t_bytes(ent)); 1596 blk_log_dump_pdu(s, ent, has_cg); 1597 trace_seq_printf(s, "[%s]\n", cmd); 1598 } else { 1599 if (t_sec(ent)) 1600 trace_seq_printf(s, "%llu + %u [%s]\n", 1601 t_sector(ent), t_sec(ent), cmd); 1602 else 1603 trace_seq_printf(s, "[%s]\n", cmd); 1604 } 1605} 1606 1607static void blk_log_with_error(struct trace_seq *s, 1608 const struct trace_entry *ent, bool has_cg) 1609{ 1610 if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) { 1611 blk_log_dump_pdu(s, ent, has_cg); 1612 trace_seq_printf(s, "[%d]\n", t_error(ent)); 1613 } else { 1614 if (t_sec(ent)) 1615 trace_seq_printf(s, "%llu + %u [%d]\n", 1616 t_sector(ent), 1617 t_sec(ent), t_error(ent)); 1618 else 1619 trace_seq_printf(s, "%llu [%d]\n", 1620 t_sector(ent), t_error(ent)); 1621 } 1622} 1623 1624static void blk_log_remap(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 1625{ 1626 const struct blk_io_trace_remap *__r = pdu_start(ent, has_cg); 1627 1628 trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n", 1629 t_sector(ent), t_sec(ent), 1630 MAJOR(be32_to_cpu(__r->device_from)), 1631 MINOR(be32_to_cpu(__r->device_from)), 1632 be64_to_cpu(__r->sector_from)); 1633} 1634 1635static void blk_log_plug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 1636{ 1637 char cmd[TASK_COMM_LEN]; 1638 1639 trace_find_cmdline(ent->pid, cmd); 1640 1641 trace_seq_printf(s, "[%s]\n", cmd); 1642} 1643 1644static void blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 1645{ 1646 char cmd[TASK_COMM_LEN]; 1647 1648 trace_find_cmdline(ent->pid, cmd); 1649 1650 trace_seq_printf(s, "[%s] %llu\n", cmd, get_pdu_int(ent, has_cg)); 1651} 1652 1653static void blk_log_split(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 1654{ 1655 char cmd[TASK_COMM_LEN]; 1656 1657 trace_find_cmdline(ent->pid, cmd); 1658 1659 trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent), 1660 get_pdu_int(ent, has_cg), cmd); 1661} 1662 1663static void blk_log_msg(struct trace_seq *s, const struct trace_entry *ent, 1664 bool has_cg) 1665{ 1666 1667 trace_seq_putmem(s, pdu_start(ent, has_cg), 1668 pdu_real_len(ent, has_cg)); 1669 trace_seq_putc(s, '\n'); 1670} 1671 1672/* 1673 * struct tracer operations 1674 */ 1675 1676static void blk_tracer_print_header(struct seq_file *m) 1677{ 1678 if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC)) 1679 return; 1680 seq_puts(m, "# DEV CPU TIMESTAMP PID ACT FLG\n" 1681 "# | | | | | |\n"); 1682} 1683 1684static void blk_tracer_start(struct trace_array *tr) 1685{ 1686 blk_tracer_enabled = true; 1687} 1688 1689static int blk_tracer_init(struct trace_array *tr) 1690{ 1691 blk_tr = tr; 1692 blk_tracer_start(tr); 1693 return 0; 1694} 1695 1696static void blk_tracer_stop(struct trace_array *tr) 1697{ 1698 blk_tracer_enabled = false; 1699} 1700 1701static void blk_tracer_reset(struct trace_array *tr) 1702{ 1703 blk_tracer_stop(tr); 1704} 1705 1706static const struct { 1707 const char *act[2]; 1708 void (*print)(struct trace_seq *s, const struct trace_entry *ent, 1709 bool has_cg); 1710} what2act[] = { 1711 [__BLK_TA_QUEUE] = {{ "Q", "queue" }, blk_log_generic }, 1712 [__BLK_TA_BACKMERGE] = {{ "M", "backmerge" }, blk_log_generic }, 1713 [__BLK_TA_FRONTMERGE] = {{ "F", "frontmerge" }, blk_log_generic }, 1714 [__BLK_TA_GETRQ] = {{ "G", "getrq" }, blk_log_generic }, 1715 [__BLK_TA_SLEEPRQ] = {{ "S", "sleeprq" }, blk_log_generic }, 1716 [__BLK_TA_REQUEUE] = {{ "R", "requeue" }, blk_log_with_error }, 1717 [__BLK_TA_ISSUE] = {{ "D", "issue" }, blk_log_generic }, 1718 [__BLK_TA_COMPLETE] = {{ "C", "complete" }, blk_log_with_error }, 1719 [__BLK_TA_PLUG] = {{ "P", "plug" }, blk_log_plug }, 1720 [__BLK_TA_UNPLUG_IO] = {{ "U", "unplug_io" }, blk_log_unplug }, 1721 [__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug }, 1722 [__BLK_TA_INSERT] = {{ "I", "insert" }, blk_log_generic }, 1723 [__BLK_TA_SPLIT] = {{ "X", "split" }, blk_log_split }, 1724 [__BLK_TA_REMAP] = {{ "A", "remap" }, blk_log_remap }, 1725}; 1726 1727static enum print_line_t print_one_line(struct trace_iterator *iter, 1728 bool classic) 1729{ 1730 struct trace_array *tr = iter->tr; 1731 struct trace_seq *s = &iter->seq; 1732 const struct blk_io_trace2 *t; 1733 u16 what; 1734 bool long_act; 1735 blk_log_action_t *log_action; 1736 bool has_cg; 1737 1738 t = te_blk_io_trace(iter->ent); 1739 what = (t->action & ((1 << BLK_TC_SHIFT) - 1)) & ~__BLK_TA_CGROUP; 1740 long_act = !!(tr->trace_flags & TRACE_ITER(VERBOSE)); 1741 log_action = classic ? &blk_log_action_classic : &blk_log_action; 1742 has_cg = t->action & __BLK_TA_CGROUP; 1743 1744 if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) { 1745 log_action(iter, long_act ? "message" : "m", has_cg); 1746 blk_log_msg(s, iter->ent, has_cg); 1747 return trace_handle_return(s); 1748 } 1749 1750 if (unlikely(what == 0 || what >= ARRAY_SIZE(what2act))) 1751 trace_seq_printf(s, "Unknown action %x\n", what); 1752 else { 1753 log_action(iter, what2act[what].act[long_act], has_cg); 1754 what2act[what].print(s, iter->ent, has_cg); 1755 } 1756 1757 return trace_handle_return(s); 1758} 1759 1760static enum print_line_t blk_trace_event_print(struct trace_iterator *iter, 1761 int flags, struct trace_event *event) 1762{ 1763 return print_one_line(iter, false); 1764} 1765 1766static void blk_trace_synthesize_old_trace(struct trace_iterator *iter) 1767{ 1768 struct trace_seq *s = &iter->seq; 1769 struct blk_io_trace2 *t = (struct blk_io_trace2 *)iter->ent; 1770 const int offset = offsetof(struct blk_io_trace2, sector); 1771 struct blk_io_trace old = { 1772 .magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION, 1773 .time = iter->ts, 1774 }; 1775 1776 trace_seq_putmem(s, &old, offset); 1777 trace_seq_putmem(s, &t->sector, 1778 sizeof(old) - offset + t->pdu_len); 1779} 1780 1781static enum print_line_t 1782blk_trace_event_print_binary(struct trace_iterator *iter, int flags, 1783 struct trace_event *event) 1784{ 1785 blk_trace_synthesize_old_trace(iter); 1786 1787 return trace_handle_return(&iter->seq); 1788} 1789 1790static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter) 1791{ 1792 if ((iter->ent->type != TRACE_BLK) || 1793 !(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC)) 1794 return TRACE_TYPE_UNHANDLED; 1795 1796 return print_one_line(iter, true); 1797} 1798 1799static int 1800blk_tracer_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 1801{ 1802 /* don't output context-info for blk_classic output */ 1803 if (bit == TRACE_BLK_OPT_CLASSIC) { 1804 if (set) 1805 tr->trace_flags &= ~TRACE_ITER(CONTEXT_INFO); 1806 else 1807 tr->trace_flags |= TRACE_ITER(CONTEXT_INFO); 1808 } 1809 return 0; 1810} 1811 1812static struct tracer blk_tracer __read_mostly = { 1813 .name = "blk", 1814 .init = blk_tracer_init, 1815 .reset = blk_tracer_reset, 1816 .start = blk_tracer_start, 1817 .stop = blk_tracer_stop, 1818 .print_header = blk_tracer_print_header, 1819 .print_line = blk_tracer_print_line, 1820 .flags = &blk_tracer_flags, 1821 .set_flag = blk_tracer_set_flag, 1822}; 1823 1824static struct trace_event_functions trace_blk_event_funcs = { 1825 .trace = blk_trace_event_print, 1826 .binary = blk_trace_event_print_binary, 1827}; 1828 1829static struct trace_event trace_blk_event = { 1830 .type = TRACE_BLK, 1831 .funcs = &trace_blk_event_funcs, 1832}; 1833 1834static int __init init_blk_tracer(void) 1835{ 1836 if (!register_trace_event(&trace_blk_event)) { 1837 pr_warn("Warning: could not register block events\n"); 1838 return 1; 1839 } 1840 1841 if (register_tracer(&blk_tracer) != 0) { 1842 pr_warn("Warning: could not register the block tracer\n"); 1843 unregister_trace_event(&trace_blk_event); 1844 return 1; 1845 } 1846 1847 BUILD_BUG_ON(__alignof__(struct blk_user_trace_setup2) % 1848 __alignof__(long)); 1849 BUILD_BUG_ON(__alignof__(struct blk_io_trace2) % __alignof__(long)); 1850 1851 return 0; 1852} 1853 1854device_initcall(init_blk_tracer); 1855 1856static int blk_trace_remove_queue(struct request_queue *q) 1857{ 1858 struct blk_trace *bt; 1859 1860 bt = rcu_replace_pointer(q->blk_trace, NULL, 1861 lockdep_is_held(&q->debugfs_mutex)); 1862 if (bt == NULL) 1863 return -EINVAL; 1864 1865 blk_trace_stop(bt); 1866 1867 put_probe_ref(); 1868 synchronize_rcu(); 1869 blk_trace_free(q, bt); 1870 return 0; 1871} 1872 1873/* 1874 * Setup everything required to start tracing 1875 */ 1876static int blk_trace_setup_queue(struct request_queue *q, 1877 struct block_device *bdev) 1878{ 1879 struct blk_trace *bt = NULL; 1880 int ret = -ENOMEM; 1881 1882 bt = kzalloc(sizeof(*bt), GFP_KERNEL); 1883 if (!bt) 1884 return -ENOMEM; 1885 1886 bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char)); 1887 if (!bt->msg_data) 1888 goto free_bt; 1889 1890 bt->dev = bdev->bd_dev; 1891 bt->act_mask = (u16)-1; 1892 1893 blk_trace_setup_lba(bt, bdev); 1894 1895 rcu_assign_pointer(q->blk_trace, bt); 1896 get_probe_ref(); 1897 return 0; 1898 1899free_bt: 1900 blk_trace_free(q, bt); 1901 return ret; 1902} 1903 1904/* 1905 * sysfs interface to enable and configure tracing 1906 */ 1907 1908static ssize_t sysfs_blk_trace_attr_show(struct device *dev, 1909 struct device_attribute *attr, 1910 char *buf); 1911static ssize_t sysfs_blk_trace_attr_store(struct device *dev, 1912 struct device_attribute *attr, 1913 const char *buf, size_t count); 1914#define BLK_TRACE_DEVICE_ATTR(_name) \ 1915 DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \ 1916 sysfs_blk_trace_attr_show, \ 1917 sysfs_blk_trace_attr_store) 1918 1919static BLK_TRACE_DEVICE_ATTR(enable); 1920static BLK_TRACE_DEVICE_ATTR(act_mask); 1921static BLK_TRACE_DEVICE_ATTR(pid); 1922static BLK_TRACE_DEVICE_ATTR(start_lba); 1923static BLK_TRACE_DEVICE_ATTR(end_lba); 1924 1925static struct attribute *blk_trace_attrs[] = { 1926 &dev_attr_enable.attr, 1927 &dev_attr_act_mask.attr, 1928 &dev_attr_pid.attr, 1929 &dev_attr_start_lba.attr, 1930 &dev_attr_end_lba.attr, 1931 NULL 1932}; 1933 1934struct attribute_group blk_trace_attr_group = { 1935 .name = "trace", 1936 .attrs = blk_trace_attrs, 1937}; 1938 1939static const struct { 1940 int mask; 1941 const char *str; 1942} mask_maps[] = { 1943 { BLK_TC_READ, "read" }, 1944 { BLK_TC_WRITE, "write" }, 1945 { BLK_TC_FLUSH, "flush" }, 1946 { BLK_TC_SYNC, "sync" }, 1947 { BLK_TC_QUEUE, "queue" }, 1948 { BLK_TC_REQUEUE, "requeue" }, 1949 { BLK_TC_ISSUE, "issue" }, 1950 { BLK_TC_COMPLETE, "complete" }, 1951 { BLK_TC_FS, "fs" }, 1952 { BLK_TC_PC, "pc" }, 1953 { BLK_TC_NOTIFY, "notify" }, 1954 { BLK_TC_AHEAD, "ahead" }, 1955 { BLK_TC_META, "meta" }, 1956 { BLK_TC_DISCARD, "discard" }, 1957 { BLK_TC_DRV_DATA, "drv_data" }, 1958 { BLK_TC_FUA, "fua" }, 1959 { BLK_TC_WRITE_ZEROES, "write-zeroes" }, 1960}; 1961 1962static int blk_trace_str2mask(const char *str) 1963{ 1964 int i; 1965 int mask = 0; 1966 char *buf, *s, *token; 1967 1968 buf = kstrdup(str, GFP_KERNEL); 1969 if (buf == NULL) 1970 return -ENOMEM; 1971 s = strstrip(buf); 1972 1973 while (1) { 1974 token = strsep(&s, ","); 1975 if (token == NULL) 1976 break; 1977 1978 if (*token == '\0') 1979 continue; 1980 1981 for (i = 0; i < ARRAY_SIZE(mask_maps); i++) { 1982 if (strcasecmp(token, mask_maps[i].str) == 0) { 1983 mask |= mask_maps[i].mask; 1984 break; 1985 } 1986 } 1987 if (i == ARRAY_SIZE(mask_maps)) { 1988 mask = -EINVAL; 1989 break; 1990 } 1991 } 1992 kfree(buf); 1993 1994 return mask; 1995} 1996 1997static ssize_t blk_trace_mask2str(char *buf, int mask) 1998{ 1999 int i; 2000 char *p = buf; 2001 2002 for (i = 0; i < ARRAY_SIZE(mask_maps); i++) { 2003 if (mask & mask_maps[i].mask) { 2004 p += sprintf(p, "%s%s", 2005 (p == buf) ? "" : ",", mask_maps[i].str); 2006 } 2007 } 2008 *p++ = '\n'; 2009 2010 return p - buf; 2011} 2012 2013static ssize_t sysfs_blk_trace_attr_show(struct device *dev, 2014 struct device_attribute *attr, 2015 char *buf) 2016{ 2017 struct block_device *bdev = dev_to_bdev(dev); 2018 struct request_queue *q = bdev_get_queue(bdev); 2019 struct blk_trace *bt; 2020 ssize_t ret = -ENXIO; 2021 2022 mutex_lock(&q->debugfs_mutex); 2023 2024 bt = rcu_dereference_protected(q->blk_trace, 2025 lockdep_is_held(&q->debugfs_mutex)); 2026 if (attr == &dev_attr_enable) { 2027 ret = sprintf(buf, "%u\n", !!bt); 2028 goto out_unlock_bdev; 2029 } 2030 2031 if (bt == NULL) 2032 ret = sprintf(buf, "disabled\n"); 2033 else if (attr == &dev_attr_act_mask) 2034 ret = blk_trace_mask2str(buf, bt->act_mask); 2035 else if (attr == &dev_attr_pid) 2036 ret = sprintf(buf, "%u\n", bt->pid); 2037 else if (attr == &dev_attr_start_lba) 2038 ret = sprintf(buf, "%llu\n", bt->start_lba); 2039 else if (attr == &dev_attr_end_lba) 2040 ret = sprintf(buf, "%llu\n", bt->end_lba); 2041 2042out_unlock_bdev: 2043 mutex_unlock(&q->debugfs_mutex); 2044 return ret; 2045} 2046 2047static ssize_t sysfs_blk_trace_attr_store(struct device *dev, 2048 struct device_attribute *attr, 2049 const char *buf, size_t count) 2050{ 2051 struct block_device *bdev = dev_to_bdev(dev); 2052 struct request_queue *q = bdev_get_queue(bdev); 2053 struct blk_trace *bt; 2054 u64 value; 2055 ssize_t ret = -EINVAL; 2056 2057 if (count == 0) 2058 goto out; 2059 2060 if (attr == &dev_attr_act_mask) { 2061 if (kstrtoull(buf, 0, &value)) { 2062 /* Assume it is a list of trace category names */ 2063 ret = blk_trace_str2mask(buf); 2064 if (ret < 0) 2065 goto out; 2066 value = ret; 2067 } 2068 } else { 2069 if (kstrtoull(buf, 0, &value)) 2070 goto out; 2071 } 2072 2073 mutex_lock(&q->debugfs_mutex); 2074 2075 bt = rcu_dereference_protected(q->blk_trace, 2076 lockdep_is_held(&q->debugfs_mutex)); 2077 if (attr == &dev_attr_enable) { 2078 if (!!value == !!bt) { 2079 ret = 0; 2080 goto out_unlock_bdev; 2081 } 2082 if (value) 2083 ret = blk_trace_setup_queue(q, bdev); 2084 else 2085 ret = blk_trace_remove_queue(q); 2086 goto out_unlock_bdev; 2087 } 2088 2089 ret = 0; 2090 if (bt == NULL) { 2091 ret = blk_trace_setup_queue(q, bdev); 2092 bt = rcu_dereference_protected(q->blk_trace, 2093 lockdep_is_held(&q->debugfs_mutex)); 2094 } 2095 2096 if (ret == 0) { 2097 if (attr == &dev_attr_act_mask) 2098 bt->act_mask = value; 2099 else if (attr == &dev_attr_pid) 2100 bt->pid = value; 2101 else if (attr == &dev_attr_start_lba) 2102 bt->start_lba = value; 2103 else if (attr == &dev_attr_end_lba) 2104 bt->end_lba = value; 2105 } 2106 2107out_unlock_bdev: 2108 mutex_unlock(&q->debugfs_mutex); 2109out: 2110 return ret ? ret : count; 2111} 2112#endif /* CONFIG_BLK_DEV_IO_TRACE */ 2113 2114#ifdef CONFIG_EVENT_TRACING 2115 2116/** 2117 * blk_fill_rwbs - Fill the buffer rwbs by mapping op to character string. 2118 * @rwbs: buffer to be filled 2119 * @opf: request operation type (REQ_OP_XXX) and flags for the tracepoint 2120 * 2121 * Description: 2122 * Maps each request operation and flag to a single character and fills the 2123 * buffer provided by the caller with resulting string. 2124 * 2125 **/ 2126void blk_fill_rwbs(char *rwbs, blk_opf_t opf) 2127{ 2128 int i = 0; 2129 2130 if (opf & REQ_PREFLUSH) 2131 rwbs[i++] = 'F'; 2132 2133 switch (opf & REQ_OP_MASK) { 2134 case REQ_OP_WRITE: 2135 rwbs[i++] = 'W'; 2136 break; 2137 case REQ_OP_DISCARD: 2138 rwbs[i++] = 'D'; 2139 break; 2140 case REQ_OP_SECURE_ERASE: 2141 rwbs[i++] = 'D'; 2142 rwbs[i++] = 'E'; 2143 break; 2144 case REQ_OP_FLUSH: 2145 rwbs[i++] = 'F'; 2146 break; 2147 case REQ_OP_READ: 2148 rwbs[i++] = 'R'; 2149 break; 2150 case REQ_OP_ZONE_APPEND: 2151 rwbs[i++] = 'Z'; 2152 rwbs[i++] = 'A'; 2153 break; 2154 case REQ_OP_ZONE_RESET: 2155 case REQ_OP_ZONE_RESET_ALL: 2156 rwbs[i++] = 'Z'; 2157 rwbs[i++] = 'R'; 2158 if ((opf & REQ_OP_MASK) == REQ_OP_ZONE_RESET_ALL) 2159 rwbs[i++] = 'A'; 2160 break; 2161 case REQ_OP_ZONE_FINISH: 2162 rwbs[i++] = 'Z'; 2163 rwbs[i++] = 'F'; 2164 break; 2165 case REQ_OP_ZONE_OPEN: 2166 rwbs[i++] = 'Z'; 2167 rwbs[i++] = 'O'; 2168 break; 2169 case REQ_OP_ZONE_CLOSE: 2170 rwbs[i++] = 'Z'; 2171 rwbs[i++] = 'C'; 2172 break; 2173 case REQ_OP_WRITE_ZEROES: 2174 rwbs[i++] = 'W'; 2175 rwbs[i++] = 'Z'; 2176 break; 2177 default: 2178 rwbs[i++] = 'N'; 2179 } 2180 2181 if (opf & REQ_FUA) 2182 rwbs[i++] = 'F'; 2183 if (opf & REQ_RAHEAD) 2184 rwbs[i++] = 'A'; 2185 if (opf & REQ_SYNC) 2186 rwbs[i++] = 'S'; 2187 if (opf & REQ_META) 2188 rwbs[i++] = 'M'; 2189 if (opf & REQ_ATOMIC) 2190 rwbs[i++] = 'U'; 2191 2192 WARN_ON_ONCE(i >= RWBS_LEN); 2193 2194 rwbs[i] = '\0'; 2195} 2196EXPORT_SYMBOL_GPL(blk_fill_rwbs); 2197 2198#endif /* CONFIG_EVENT_TRACING */ 2199
~/
Read-only file preview.