FILE
block.c
File preview with syntax highlighting.
linux-6.19.14/drivers/mmc/core/block.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Block driver for media (i.e., flash cards) 4 * 5 * Copyright 2002 Hewlett-Packard Company 6 * Copyright 2005-2008 Pierre Ossman 7 * 8 * Use consistent with the GNU GPL is permitted, 9 * provided that this copyright notice is 10 * preserved in its entirety in all copies and derived works. 11 * 12 * HEWLETT-PACKARD COMPANY MAKES NO WARRANTIES, EXPRESSED OR IMPLIED, 13 * AS TO THE USEFULNESS OR CORRECTNESS OF THIS CODE OR ITS 14 * FITNESS FOR ANY PARTICULAR PURPOSE. 15 * 16 * Many thanks to Alessandro Rubini and Jonathan Corbet! 17 * 18 * Author: Andrew Christian 19 * 28 May 2002 20 */ 21#include <linux/moduleparam.h> 22#include <linux/module.h> 23#include <linux/init.h> 24 25#include <linux/kernel.h> 26#include <linux/fs.h> 27#include <linux/slab.h> 28#include <linux/errno.h> 29#include <linux/hdreg.h> 30#include <linux/kdev_t.h> 31#include <linux/kref.h> 32#include <linux/blkdev.h> 33#include <linux/cdev.h> 34#include <linux/mutex.h> 35#include <linux/scatterlist.h> 36#include <linux/string.h> 37#include <linux/string_helpers.h> 38#include <linux/delay.h> 39#include <linux/capability.h> 40#include <linux/compat.h> 41#include <linux/pm_runtime.h> 42#include <linux/idr.h> 43#include <linux/debugfs.h> 44#include <linux/rpmb.h> 45 46#include <linux/mmc/ioctl.h> 47#include <linux/mmc/card.h> 48#include <linux/mmc/host.h> 49#include <linux/mmc/mmc.h> 50#include <linux/mmc/sd.h> 51 52#include <linux/uaccess.h> 53#include <linux/unaligned.h> 54 55#include "queue.h" 56#include "block.h" 57#include "core.h" 58#include "card.h" 59#include "crypto.h" 60#include "host.h" 61#include "bus.h" 62#include "mmc_ops.h" 63#include "quirks.h" 64#include "sd_ops.h" 65 66MODULE_ALIAS("mmc:block"); 67#ifdef MODULE_PARAM_PREFIX 68#undef MODULE_PARAM_PREFIX 69#endif 70#define MODULE_PARAM_PREFIX "mmcblk." 71 72/* 73 * Set a 10 second timeout for polling write request busy state. Note, mmc core 74 * is setting a 3 second timeout for SD cards, and SDHCI has long had a 10 75 * second software timer to timeout the whole request, so 10 seconds should be 76 * ample. 77 */ 78#define MMC_BLK_TIMEOUT_MS (10 * 1000) 79#define MMC_EXTRACT_INDEX_FROM_ARG(x) ((x & 0x00FF0000) >> 16) 80#define MMC_EXTRACT_VALUE_FROM_ARG(x) ((x & 0x0000FF00) >> 8) 81 82#define RPMB_FRAME_SIZE sizeof(struct rpmb_frame) 83#define CHECK_SIZE_NEQ(val) ((val) != sizeof(struct rpmb_frame)) 84#define CHECK_SIZE_ALIGNED(val) IS_ALIGNED((val), sizeof(struct rpmb_frame)) 85 86static DEFINE_MUTEX(block_mutex); 87 88/* 89 * The defaults come from config options but can be overriden by module 90 * or bootarg options. 91 */ 92static int perdev_minors = CONFIG_MMC_BLOCK_MINORS; 93 94/* 95 * We've only got one major, so number of mmcblk devices is 96 * limited to (1 << 20) / number of minors per device. It is also 97 * limited by the MAX_DEVICES below. 98 */ 99static int max_devices; 100 101#define MAX_DEVICES 256 102 103static DEFINE_IDA(mmc_blk_ida); 104static DEFINE_IDA(mmc_rpmb_ida); 105 106struct mmc_blk_busy_data { 107 struct mmc_card *card; 108 u32 status; 109}; 110 111/* 112 * There is one mmc_blk_data per slot. 113 */ 114struct mmc_blk_data { 115 struct device *parent; 116 struct gendisk *disk; 117 struct mmc_queue queue; 118 struct list_head part; 119 struct list_head rpmbs; 120 121 unsigned int flags; 122#define MMC_BLK_CMD23 (1 << 0) /* Can do SET_BLOCK_COUNT for multiblock */ 123#define MMC_BLK_REL_WR (1 << 1) /* MMC Reliable write support */ 124 125 struct kref kref; 126 unsigned int read_only; 127 unsigned int part_type; 128 unsigned int reset_done; 129#define MMC_BLK_READ BIT(0) 130#define MMC_BLK_WRITE BIT(1) 131#define MMC_BLK_DISCARD BIT(2) 132#define MMC_BLK_SECDISCARD BIT(3) 133#define MMC_BLK_CQE_RECOVERY BIT(4) 134#define MMC_BLK_TRIM BIT(5) 135 136 /* 137 * Only set in main mmc_blk_data associated 138 * with mmc_card with dev_set_drvdata, and keeps 139 * track of the current selected device partition. 140 */ 141 unsigned int part_curr; 142#define MMC_BLK_PART_INVALID UINT_MAX /* Unknown partition active */ 143 int area_type; 144 145 /* debugfs files (only in main mmc_blk_data) */ 146 struct dentry *status_dentry; 147 struct dentry *ext_csd_dentry; 148}; 149 150/* Device type for RPMB character devices */ 151static dev_t mmc_rpmb_devt; 152 153/* Bus type for RPMB character devices */ 154static const struct bus_type mmc_rpmb_bus_type = { 155 .name = "mmc_rpmb", 156}; 157 158/** 159 * struct mmc_rpmb_data - special RPMB device type for these areas 160 * @dev: the device for the RPMB area 161 * @chrdev: character device for the RPMB area 162 * @id: unique device ID number 163 * @part_index: partition index (0 on first) 164 * @md: parent MMC block device 165 * @rdev: registered RPMB device 166 * @node: list item, so we can put this device on a list 167 */ 168struct mmc_rpmb_data { 169 struct device dev; 170 struct cdev chrdev; 171 int id; 172 unsigned int part_index; 173 struct mmc_blk_data *md; 174 struct rpmb_dev *rdev; 175 struct list_head node; 176}; 177 178static DEFINE_MUTEX(open_lock); 179 180module_param(perdev_minors, int, 0444); 181MODULE_PARM_DESC(perdev_minors, "Minors numbers to allocate per device"); 182 183static inline int mmc_blk_part_switch(struct mmc_card *card, 184 unsigned int part_type); 185static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq, 186 struct mmc_card *card, 187 int recovery_mode, 188 struct mmc_queue *mq); 189static void mmc_blk_hsq_req_done(struct mmc_request *mrq); 190static int mmc_spi_err_check(struct mmc_card *card); 191static int mmc_blk_busy_cb(void *cb_data, bool *busy); 192 193static struct mmc_blk_data *mmc_blk_get(struct gendisk *disk) 194{ 195 struct mmc_blk_data *md; 196 197 mutex_lock(&open_lock); 198 md = disk->private_data; 199 if (md && !kref_get_unless_zero(&md->kref)) 200 md = NULL; 201 mutex_unlock(&open_lock); 202 203 return md; 204} 205 206static inline int mmc_get_devidx(struct gendisk *disk) 207{ 208 int devidx = disk->first_minor / perdev_minors; 209 return devidx; 210} 211 212static void mmc_blk_kref_release(struct kref *ref) 213{ 214 struct mmc_blk_data *md = container_of(ref, struct mmc_blk_data, kref); 215 int devidx; 216 217 devidx = mmc_get_devidx(md->disk); 218 ida_free(&mmc_blk_ida, devidx); 219 220 mutex_lock(&open_lock); 221 md->disk->private_data = NULL; 222 mutex_unlock(&open_lock); 223 224 put_disk(md->disk); 225 kfree(md); 226} 227 228static void mmc_blk_put(struct mmc_blk_data *md) 229{ 230 kref_put(&md->kref, mmc_blk_kref_release); 231} 232 233static ssize_t power_ro_lock_show(struct device *dev, 234 struct device_attribute *attr, char *buf) 235{ 236 int ret; 237 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); 238 struct mmc_card *card = md->queue.card; 239 int locked = 0; 240 241 if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PERM_WP_EN) 242 locked = 2; 243 else if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PWR_WP_EN) 244 locked = 1; 245 246 ret = sysfs_emit(buf, "%d\n", locked); 247 248 mmc_blk_put(md); 249 250 return ret; 251} 252 253static ssize_t power_ro_lock_store(struct device *dev, 254 struct device_attribute *attr, const char *buf, size_t count) 255{ 256 int ret; 257 struct mmc_blk_data *md, *part_md; 258 struct mmc_queue *mq; 259 struct request *req; 260 unsigned long set; 261 262 if (kstrtoul(buf, 0, &set)) 263 return -EINVAL; 264 265 if (set != 1) 266 return count; 267 268 md = mmc_blk_get(dev_to_disk(dev)); 269 mq = &md->queue; 270 271 /* Dispatch locking to the block layer */ 272 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_OUT, 0); 273 if (IS_ERR(req)) { 274 count = PTR_ERR(req); 275 goto out_put; 276 } 277 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP; 278 req_to_mmc_queue_req(req)->drv_op_result = -EIO; 279 blk_execute_rq(req, false); 280 ret = req_to_mmc_queue_req(req)->drv_op_result; 281 blk_mq_free_request(req); 282 283 if (!ret) { 284 pr_info("%s: Locking boot partition ro until next power on\n", 285 md->disk->disk_name); 286 set_disk_ro(md->disk, 1); 287 288 list_for_each_entry(part_md, &md->part, part) 289 if (part_md->area_type == MMC_BLK_DATA_AREA_BOOT) { 290 pr_info("%s: Locking boot partition ro until next power on\n", part_md->disk->disk_name); 291 set_disk_ro(part_md->disk, 1); 292 } 293 } 294out_put: 295 mmc_blk_put(md); 296 return count; 297} 298 299static DEVICE_ATTR(ro_lock_until_next_power_on, 0, 300 power_ro_lock_show, power_ro_lock_store); 301 302static ssize_t force_ro_show(struct device *dev, struct device_attribute *attr, 303 char *buf) 304{ 305 int ret; 306 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); 307 308 ret = sysfs_emit(buf, "%d\n", 309 get_disk_ro(dev_to_disk(dev)) ^ 310 md->read_only); 311 mmc_blk_put(md); 312 return ret; 313} 314 315static ssize_t force_ro_store(struct device *dev, struct device_attribute *attr, 316 const char *buf, size_t count) 317{ 318 int ret; 319 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); 320 unsigned long set; 321 322 if (kstrtoul(buf, 0, &set)) { 323 ret = -EINVAL; 324 goto out; 325 } 326 327 set_disk_ro(dev_to_disk(dev), set || md->read_only); 328 ret = count; 329out: 330 mmc_blk_put(md); 331 return ret; 332} 333 334static DEVICE_ATTR(force_ro, 0644, force_ro_show, force_ro_store); 335 336static struct attribute *mmc_disk_attrs[] = { 337 &dev_attr_force_ro.attr, 338 &dev_attr_ro_lock_until_next_power_on.attr, 339 NULL, 340}; 341 342static umode_t mmc_disk_attrs_is_visible(struct kobject *kobj, 343 struct attribute *a, int n) 344{ 345 struct device *dev = kobj_to_dev(kobj); 346 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); 347 umode_t mode = a->mode; 348 349 if (a == &dev_attr_ro_lock_until_next_power_on.attr && 350 (md->area_type & MMC_BLK_DATA_AREA_BOOT) && 351 md->queue.card->ext_csd.boot_ro_lockable) { 352 mode = 0444; 353 if (!(md->queue.card->ext_csd.boot_ro_lock & 354 EXT_CSD_BOOT_WP_B_PWR_WP_DIS)) 355 mode |= 0200; 356 } 357 358 mmc_blk_put(md); 359 return mode; 360} 361 362static const struct attribute_group mmc_disk_attr_group = { 363 .is_visible = mmc_disk_attrs_is_visible, 364 .attrs = mmc_disk_attrs, 365}; 366 367static const struct attribute_group *mmc_disk_attr_groups[] = { 368 &mmc_disk_attr_group, 369 NULL, 370}; 371 372static int mmc_blk_open(struct gendisk *disk, blk_mode_t mode) 373{ 374 struct mmc_blk_data *md = mmc_blk_get(disk); 375 int ret = -ENXIO; 376 377 mutex_lock(&block_mutex); 378 if (md) { 379 ret = 0; 380 if ((mode & BLK_OPEN_WRITE) && md->read_only) { 381 mmc_blk_put(md); 382 ret = -EROFS; 383 } 384 } 385 mutex_unlock(&block_mutex); 386 387 return ret; 388} 389 390static void mmc_blk_release(struct gendisk *disk) 391{ 392 struct mmc_blk_data *md = disk->private_data; 393 394 mutex_lock(&block_mutex); 395 mmc_blk_put(md); 396 mutex_unlock(&block_mutex); 397} 398 399static int 400mmc_blk_getgeo(struct gendisk *disk, struct hd_geometry *geo) 401{ 402 geo->cylinders = get_capacity(disk) / (4 * 16); 403 geo->heads = 4; 404 geo->sectors = 16; 405 return 0; 406} 407 408struct mmc_blk_ioc_data { 409 struct mmc_ioc_cmd ic; 410 unsigned char *buf; 411 u64 buf_bytes; 412 unsigned int flags; 413#define MMC_BLK_IOC_DROP BIT(0) /* drop this mrq */ 414#define MMC_BLK_IOC_SBC BIT(1) /* use mrq.sbc */ 415 416 struct mmc_rpmb_data *rpmb; 417}; 418 419static struct mmc_blk_ioc_data *mmc_blk_ioctl_copy_from_user( 420 struct mmc_ioc_cmd __user *user) 421{ 422 struct mmc_blk_ioc_data *idata; 423 int err; 424 425 idata = kzalloc(sizeof(*idata), GFP_KERNEL); 426 if (!idata) { 427 err = -ENOMEM; 428 goto out; 429 } 430 431 if (copy_from_user(&idata->ic, user, sizeof(idata->ic))) { 432 err = -EFAULT; 433 goto idata_err; 434 } 435 436 idata->buf_bytes = (u64) idata->ic.blksz * idata->ic.blocks; 437 if (idata->buf_bytes > MMC_IOC_MAX_BYTES) { 438 err = -EOVERFLOW; 439 goto idata_err; 440 } 441 442 if (!idata->buf_bytes) { 443 idata->buf = NULL; 444 return idata; 445 } 446 447 idata->buf = memdup_user((void __user *)(unsigned long) 448 idata->ic.data_ptr, idata->buf_bytes); 449 if (IS_ERR(idata->buf)) { 450 err = PTR_ERR(idata->buf); 451 goto idata_err; 452 } 453 454 return idata; 455 456idata_err: 457 kfree(idata); 458out: 459 return ERR_PTR(err); 460} 461 462static int mmc_blk_ioctl_copy_to_user(struct mmc_ioc_cmd __user *ic_ptr, 463 struct mmc_blk_ioc_data *idata) 464{ 465 struct mmc_ioc_cmd *ic = &idata->ic; 466 467 if (copy_to_user(&(ic_ptr->response), ic->response, 468 sizeof(ic->response))) 469 return -EFAULT; 470 471 if (!idata->ic.write_flag) { 472 if (copy_to_user((void __user *)(unsigned long)ic->data_ptr, 473 idata->buf, idata->buf_bytes)) 474 return -EFAULT; 475 } 476 477 return 0; 478} 479 480static int __mmc_blk_ioctl_cmd(struct mmc_card *card, struct mmc_blk_data *md, 481 struct mmc_blk_ioc_data **idatas, int i) 482{ 483 struct mmc_command cmd = {}, sbc = {}; 484 struct mmc_data data = {}; 485 struct mmc_request mrq = {}; 486 struct scatterlist sg; 487 bool r1b_resp; 488 unsigned int busy_timeout_ms; 489 int err; 490 unsigned int target_part; 491 struct mmc_blk_ioc_data *idata = idatas[i]; 492 struct mmc_blk_ioc_data *prev_idata = NULL; 493 494 if (!card || !md || !idata) 495 return -EINVAL; 496 497 if (idata->flags & MMC_BLK_IOC_DROP) 498 return 0; 499 500 if (idata->flags & MMC_BLK_IOC_SBC && i > 0) 501 prev_idata = idatas[i - 1]; 502 503 /* 504 * The RPMB accesses comes in from the character device, so we 505 * need to target these explicitly. Else we just target the 506 * partition type for the block device the ioctl() was issued 507 * on. 508 */ 509 if (idata->rpmb) { 510 /* Support multiple RPMB partitions */ 511 target_part = idata->rpmb->part_index; 512 target_part |= EXT_CSD_PART_CONFIG_ACC_RPMB; 513 } else { 514 target_part = md->part_type; 515 } 516 517 cmd.opcode = idata->ic.opcode; 518 cmd.arg = idata->ic.arg; 519 cmd.flags = idata->ic.flags; 520 521 if (idata->buf_bytes) { 522 data.sg = &sg; 523 data.sg_len = 1; 524 data.blksz = idata->ic.blksz; 525 data.blocks = idata->ic.blocks; 526 527 sg_init_one(data.sg, idata->buf, idata->buf_bytes); 528 529 if (idata->ic.write_flag) 530 data.flags = MMC_DATA_WRITE; 531 else 532 data.flags = MMC_DATA_READ; 533 534 /* data.flags must already be set before doing this. */ 535 mmc_set_data_timeout(&data, card); 536 537 /* Allow overriding the timeout_ns for empirical tuning. */ 538 if (idata->ic.data_timeout_ns) 539 data.timeout_ns = idata->ic.data_timeout_ns; 540 541 mrq.data = &data; 542 } 543 544 mrq.cmd = &cmd; 545 546 err = mmc_blk_part_switch(card, target_part); 547 if (err) 548 return err; 549 550 if (idata->ic.is_acmd) { 551 err = mmc_app_cmd(card->host, card); 552 if (err) 553 return err; 554 } 555 556 if (idata->rpmb || prev_idata) { 557 sbc.opcode = MMC_SET_BLOCK_COUNT; 558 /* 559 * We don't do any blockcount validation because the max size 560 * may be increased by a future standard. We just copy the 561 * 'Reliable Write' bit here. 562 */ 563 sbc.arg = data.blocks | (idata->ic.write_flag & BIT(31)); 564 if (prev_idata) 565 sbc.arg = prev_idata->ic.arg; 566 sbc.flags = MMC_RSP_R1 | MMC_CMD_AC; 567 mrq.sbc = &sbc; 568 } 569 570 if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_SANITIZE_START) && 571 (cmd.opcode == MMC_SWITCH)) 572 return mmc_sanitize(card, idata->ic.cmd_timeout_ms); 573 574 /* If it's an R1B response we need some more preparations. */ 575 busy_timeout_ms = idata->ic.cmd_timeout_ms ? : MMC_BLK_TIMEOUT_MS; 576 r1b_resp = (cmd.flags & MMC_RSP_R1B) == MMC_RSP_R1B; 577 if (r1b_resp) 578 mmc_prepare_busy_cmd(card->host, &cmd, busy_timeout_ms); 579 580 mmc_wait_for_req(card->host, &mrq); 581 memcpy(&idata->ic.response, cmd.resp, sizeof(cmd.resp)); 582 583 if (prev_idata) { 584 memcpy(&prev_idata->ic.response, sbc.resp, sizeof(sbc.resp)); 585 if (sbc.error) { 586 dev_err(mmc_dev(card->host), "%s: sbc error %d\n", 587 __func__, sbc.error); 588 return sbc.error; 589 } 590 } 591 592 if (cmd.error) { 593 dev_err(mmc_dev(card->host), "%s: cmd error %d\n", 594 __func__, cmd.error); 595 return cmd.error; 596 } 597 if (data.error) { 598 dev_err(mmc_dev(card->host), "%s: data error %d\n", 599 __func__, data.error); 600 return data.error; 601 } 602 603 /* 604 * Make sure the cache of the PARTITION_CONFIG register and 605 * PARTITION_ACCESS bits is updated in case the ioctl ext_csd write 606 * changed it successfully. 607 */ 608 if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_PART_CONFIG) && 609 (cmd.opcode == MMC_SWITCH)) { 610 struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev); 611 u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg); 612 613 /* 614 * Update cache so the next mmc_blk_part_switch call operates 615 * on up-to-date data. 616 */ 617 card->ext_csd.part_config = value; 618 main_md->part_curr = value & EXT_CSD_PART_CONFIG_ACC_MASK; 619 } 620 621 /* 622 * Make sure to update CACHE_CTRL in case it was changed. The cache 623 * will get turned back on if the card is re-initialized, e.g. 624 * suspend/resume or hw reset in recovery. 625 */ 626 if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_CACHE_CTRL) && 627 (cmd.opcode == MMC_SWITCH)) { 628 u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg) & 1; 629 630 card->ext_csd.cache_ctrl = value; 631 } 632 633 /* 634 * According to the SD specs, some commands require a delay after 635 * issuing the command. 636 */ 637 if (idata->ic.postsleep_min_us) 638 usleep_range(idata->ic.postsleep_min_us, idata->ic.postsleep_max_us); 639 640 if (mmc_host_is_spi(card->host)) { 641 if (idata->ic.write_flag || r1b_resp || cmd.flags & MMC_RSP_SPI_BUSY) 642 return mmc_spi_err_check(card); 643 return err; 644 } 645 646 /* 647 * Ensure RPMB, writes and R1B responses are completed by polling with 648 * CMD13. Note that, usually we don't need to poll when using HW busy 649 * detection, but here it's needed since some commands may indicate the 650 * error through the R1 status bits. 651 */ 652 if (idata->rpmb || idata->ic.write_flag || r1b_resp) { 653 struct mmc_blk_busy_data cb_data = { 654 .card = card, 655 }; 656 657 err = __mmc_poll_for_busy(card->host, 0, busy_timeout_ms, 658 &mmc_blk_busy_cb, &cb_data); 659 660 idata->ic.response[0] = cb_data.status; 661 } 662 663 return err; 664} 665 666static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md, 667 struct mmc_ioc_cmd __user *ic_ptr, 668 struct mmc_rpmb_data *rpmb) 669{ 670 struct mmc_blk_ioc_data *idata; 671 struct mmc_blk_ioc_data *idatas[1]; 672 struct mmc_queue *mq; 673 struct mmc_card *card; 674 int err = 0, ioc_err = 0; 675 struct request *req; 676 677 idata = mmc_blk_ioctl_copy_from_user(ic_ptr); 678 if (IS_ERR(idata)) 679 return PTR_ERR(idata); 680 /* This will be NULL on non-RPMB ioctl():s */ 681 idata->rpmb = rpmb; 682 683 card = md->queue.card; 684 if (IS_ERR(card)) { 685 err = PTR_ERR(card); 686 goto cmd_done; 687 } 688 689 /* 690 * Dispatch the ioctl() into the block request queue. 691 */ 692 mq = &md->queue; 693 req = blk_mq_alloc_request(mq->queue, 694 idata->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0); 695 if (IS_ERR(req)) { 696 err = PTR_ERR(req); 697 goto cmd_done; 698 } 699 idatas[0] = idata; 700 req_to_mmc_queue_req(req)->drv_op = 701 rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL; 702 req_to_mmc_queue_req(req)->drv_op_result = -EIO; 703 req_to_mmc_queue_req(req)->drv_op_data = idatas; 704 req_to_mmc_queue_req(req)->ioc_count = 1; 705 blk_execute_rq(req, false); 706 ioc_err = req_to_mmc_queue_req(req)->drv_op_result; 707 err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata); 708 blk_mq_free_request(req); 709 710cmd_done: 711 kfree(idata->buf); 712 kfree(idata); 713 return ioc_err ? ioc_err : err; 714} 715 716static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md, 717 struct mmc_ioc_multi_cmd __user *user, 718 struct mmc_rpmb_data *rpmb) 719{ 720 struct mmc_blk_ioc_data **idata = NULL; 721 struct mmc_ioc_cmd __user *cmds = user->cmds; 722 struct mmc_card *card; 723 struct mmc_queue *mq; 724 int err = 0, ioc_err = 0; 725 __u64 num_of_cmds; 726 unsigned int i, n; 727 struct request *req; 728 729 if (copy_from_user(&num_of_cmds, &user->num_of_cmds, 730 sizeof(num_of_cmds))) 731 return -EFAULT; 732 733 if (!num_of_cmds) 734 return 0; 735 736 if (num_of_cmds > MMC_IOC_MAX_CMDS) 737 return -EINVAL; 738 739 n = num_of_cmds; 740 idata = kcalloc(n, sizeof(*idata), GFP_KERNEL); 741 if (!idata) 742 return -ENOMEM; 743 744 for (i = 0; i < n; i++) { 745 idata[i] = mmc_blk_ioctl_copy_from_user(&cmds[i]); 746 if (IS_ERR(idata[i])) { 747 err = PTR_ERR(idata[i]); 748 n = i; 749 goto cmd_err; 750 } 751 /* This will be NULL on non-RPMB ioctl():s */ 752 idata[i]->rpmb = rpmb; 753 } 754 755 card = md->queue.card; 756 if (IS_ERR(card)) { 757 err = PTR_ERR(card); 758 goto cmd_err; 759 } 760 761 762 /* 763 * Dispatch the ioctl()s into the block request queue. 764 */ 765 mq = &md->queue; 766 req = blk_mq_alloc_request(mq->queue, 767 idata[0]->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0); 768 if (IS_ERR(req)) { 769 err = PTR_ERR(req); 770 goto cmd_err; 771 } 772 req_to_mmc_queue_req(req)->drv_op = 773 rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL; 774 req_to_mmc_queue_req(req)->drv_op_result = -EIO; 775 req_to_mmc_queue_req(req)->drv_op_data = idata; 776 req_to_mmc_queue_req(req)->ioc_count = n; 777 blk_execute_rq(req, false); 778 ioc_err = req_to_mmc_queue_req(req)->drv_op_result; 779 780 /* copy to user if data and response */ 781 for (i = 0; i < n && !err; i++) 782 err = mmc_blk_ioctl_copy_to_user(&cmds[i], idata[i]); 783 784 blk_mq_free_request(req); 785 786cmd_err: 787 for (i = 0; i < n; i++) { 788 kfree(idata[i]->buf); 789 kfree(idata[i]); 790 } 791 kfree(idata); 792 return ioc_err ? ioc_err : err; 793} 794 795static int mmc_blk_check_blkdev(struct block_device *bdev) 796{ 797 /* 798 * The caller must have CAP_SYS_RAWIO, and must be calling this on the 799 * whole block device, not on a partition. This prevents overspray 800 * between sibling partitions. 801 */ 802 if (!capable(CAP_SYS_RAWIO) || bdev_is_partition(bdev)) 803 return -EPERM; 804 return 0; 805} 806 807static int mmc_blk_ioctl(struct block_device *bdev, blk_mode_t mode, 808 unsigned int cmd, unsigned long arg) 809{ 810 struct mmc_blk_data *md; 811 int ret; 812 813 switch (cmd) { 814 case MMC_IOC_CMD: 815 ret = mmc_blk_check_blkdev(bdev); 816 if (ret) 817 return ret; 818 md = mmc_blk_get(bdev->bd_disk); 819 if (!md) 820 return -EINVAL; 821 ret = mmc_blk_ioctl_cmd(md, 822 (struct mmc_ioc_cmd __user *)arg, 823 NULL); 824 mmc_blk_put(md); 825 return ret; 826 case MMC_IOC_MULTI_CMD: 827 ret = mmc_blk_check_blkdev(bdev); 828 if (ret) 829 return ret; 830 md = mmc_blk_get(bdev->bd_disk); 831 if (!md) 832 return -EINVAL; 833 ret = mmc_blk_ioctl_multi_cmd(md, 834 (struct mmc_ioc_multi_cmd __user *)arg, 835 NULL); 836 mmc_blk_put(md); 837 return ret; 838 default: 839 return -EINVAL; 840 } 841} 842 843#ifdef CONFIG_COMPAT 844static int mmc_blk_compat_ioctl(struct block_device *bdev, blk_mode_t mode, 845 unsigned int cmd, unsigned long arg) 846{ 847 return mmc_blk_ioctl(bdev, mode, cmd, (unsigned long) compat_ptr(arg)); 848} 849#endif 850 851static int mmc_blk_alternative_gpt_sector(struct gendisk *disk, 852 sector_t *sector) 853{ 854 struct mmc_blk_data *md; 855 int ret; 856 857 md = mmc_blk_get(disk); 858 if (!md) 859 return -EINVAL; 860 861 if (md->queue.card) 862 ret = mmc_card_alternative_gpt_sector(md->queue.card, sector); 863 else 864 ret = -ENODEV; 865 866 mmc_blk_put(md); 867 868 return ret; 869} 870 871static const struct block_device_operations mmc_bdops = { 872 .open = mmc_blk_open, 873 .release = mmc_blk_release, 874 .getgeo = mmc_blk_getgeo, 875 .owner = THIS_MODULE, 876 .ioctl = mmc_blk_ioctl, 877#ifdef CONFIG_COMPAT 878 .compat_ioctl = mmc_blk_compat_ioctl, 879#endif 880 .alternative_gpt_sector = mmc_blk_alternative_gpt_sector, 881}; 882 883static int mmc_blk_part_switch_pre(struct mmc_card *card, 884 unsigned int part_type) 885{ 886 const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_MASK; 887 const unsigned int rpmb = EXT_CSD_PART_CONFIG_ACC_RPMB; 888 int ret = 0; 889 890 if ((part_type & mask) == rpmb) { 891 if (card->ext_csd.cmdq_en) { 892 ret = mmc_cmdq_disable(card); 893 if (ret) 894 return ret; 895 } 896 mmc_retune_pause(card->host); 897 } 898 899 return ret; 900} 901 902static int mmc_blk_part_switch_post(struct mmc_card *card, 903 unsigned int part_type) 904{ 905 const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_MASK; 906 const unsigned int rpmb = EXT_CSD_PART_CONFIG_ACC_RPMB; 907 int ret = 0; 908 909 if ((part_type & mask) == rpmb) { 910 mmc_retune_unpause(card->host); 911 if (card->reenable_cmdq && !card->ext_csd.cmdq_en) 912 ret = mmc_cmdq_enable(card); 913 } 914 915 return ret; 916} 917 918static inline int mmc_blk_part_switch(struct mmc_card *card, 919 unsigned int part_type) 920{ 921 int ret = 0; 922 struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev); 923 924 if (main_md->part_curr == part_type) 925 return 0; 926 927 if (mmc_card_mmc(card)) { 928 u8 part_config = card->ext_csd.part_config; 929 930 ret = mmc_blk_part_switch_pre(card, part_type); 931 if (ret) 932 return ret; 933 934 part_config &= ~EXT_CSD_PART_CONFIG_ACC_MASK; 935 part_config |= part_type; 936 937 ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, 938 EXT_CSD_PART_CONFIG, part_config, 939 card->ext_csd.part_time); 940 if (ret) { 941 mmc_blk_part_switch_post(card, part_type); 942 return ret; 943 } 944 945 card->ext_csd.part_config = part_config; 946 947 ret = mmc_blk_part_switch_post(card, main_md->part_curr); 948 } 949 950 main_md->part_curr = part_type; 951 return ret; 952} 953 954static int mmc_sd_num_wr_blocks(struct mmc_card *card, u32 *written_blocks) 955{ 956 int err; 957 u32 result; 958 __be32 *blocks; 959 u8 resp_sz = mmc_card_ult_capacity(card) ? 8 : 4; 960 961 struct mmc_request mrq = {}; 962 struct mmc_command cmd = {}; 963 struct mmc_data data = {}; 964 struct scatterlist sg; 965 966 err = mmc_app_cmd(card->host, card); 967 if (err) 968 return err; 969 970 cmd.opcode = SD_APP_SEND_NUM_WR_BLKS; 971 cmd.arg = 0; 972 cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC; 973 974 data.blksz = resp_sz; 975 data.blocks = 1; 976 data.flags = MMC_DATA_READ; 977 data.sg = &sg; 978 data.sg_len = 1; 979 mmc_set_data_timeout(&data, card); 980 981 mrq.cmd = &cmd; 982 mrq.data = &data; 983 984 blocks = kmalloc(resp_sz, GFP_NOIO); 985 if (!blocks) 986 return -ENOMEM; 987 988 sg_init_one(&sg, blocks, resp_sz); 989 990 mmc_wait_for_req(card->host, &mrq); 991 992 if (mmc_card_ult_capacity(card)) { 993 /* 994 * Normally, ACMD22 returns the number of written sectors as 995 * u32. SDUC, however, returns it as u64. This is not a 996 * superfluous requirement, because SDUC writes may exceed 2TB. 997 * For Linux mmc however, the previously write operation could 998 * not be more than the block layer limits, thus just make room 999 * for a u64 and cast the response back to u32. 1000 */ 1001 result = clamp_val(get_unaligned_be64(blocks), 0, UINT_MAX); 1002 } else { 1003 result = ntohl(*blocks); 1004 } 1005 kfree(blocks); 1006 1007 if (cmd.error || data.error) 1008 return -EIO; 1009 1010 *written_blocks = result; 1011 1012 return 0; 1013} 1014 1015static unsigned int mmc_blk_clock_khz(struct mmc_host *host) 1016{ 1017 if (host->actual_clock) 1018 return host->actual_clock / 1000; 1019 1020 /* Clock may be subject to a divisor, fudge it by a factor of 2. */ 1021 if (host->ios.clock) 1022 return host->ios.clock / 2000; 1023 1024 /* How can there be no clock */ 1025 WARN_ON_ONCE(1); 1026 return 100; /* 100 kHz is minimum possible value */ 1027} 1028 1029static unsigned int mmc_blk_data_timeout_ms(struct mmc_host *host, 1030 struct mmc_data *data) 1031{ 1032 unsigned int ms = DIV_ROUND_UP(data->timeout_ns, 1000000); 1033 unsigned int khz; 1034 1035 if (data->timeout_clks) { 1036 khz = mmc_blk_clock_khz(host); 1037 ms += DIV_ROUND_UP(data->timeout_clks, khz); 1038 } 1039 1040 return ms; 1041} 1042 1043/* 1044 * Attempts to reset the card and get back to the requested partition. 1045 * Therefore any error here must result in cancelling the block layer 1046 * request, it must not be reattempted without going through the mmc_blk 1047 * partition sanity checks. 1048 */ 1049static int mmc_blk_reset(struct mmc_blk_data *md, struct mmc_host *host, 1050 int type) 1051{ 1052 int err; 1053 struct mmc_blk_data *main_md = dev_get_drvdata(&host->card->dev); 1054 1055 if (md->reset_done & type) 1056 return -EEXIST; 1057 1058 md->reset_done |= type; 1059 err = mmc_hw_reset(host->card); 1060 /* 1061 * A successful reset will leave the card in the main partition, but 1062 * upon failure it might not be, so set it to MMC_BLK_PART_INVALID 1063 * in that case. 1064 */ 1065 main_md->part_curr = err ? MMC_BLK_PART_INVALID : main_md->part_type; 1066 if (err) 1067 return err; 1068 /* Ensure we switch back to the correct partition */ 1069 if (mmc_blk_part_switch(host->card, md->part_type)) 1070 /* 1071 * We have failed to get back into the correct 1072 * partition, so we need to abort the whole request. 1073 */ 1074 return -ENODEV; 1075 return 0; 1076} 1077 1078static inline void mmc_blk_reset_success(struct mmc_blk_data *md, int type) 1079{ 1080 md->reset_done &= ~type; 1081} 1082 1083static void mmc_blk_check_sbc(struct mmc_queue_req *mq_rq) 1084{ 1085 struct mmc_blk_ioc_data **idata = mq_rq->drv_op_data; 1086 int i; 1087 1088 for (i = 1; i < mq_rq->ioc_count; i++) { 1089 if (idata[i - 1]->ic.opcode == MMC_SET_BLOCK_COUNT && 1090 mmc_op_multi(idata[i]->ic.opcode)) { 1091 idata[i - 1]->flags |= MMC_BLK_IOC_DROP; 1092 idata[i]->flags |= MMC_BLK_IOC_SBC; 1093 } 1094 } 1095} 1096 1097/* 1098 * The non-block commands come back from the block layer after it queued it and 1099 * processed it with all other requests and then they get issued in this 1100 * function. 1101 */ 1102static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req) 1103{ 1104 struct mmc_queue_req *mq_rq; 1105 struct mmc_card *card = mq->card; 1106 struct mmc_blk_data *md = mq->blkdata; 1107 struct mmc_blk_ioc_data **idata; 1108 bool rpmb_ioctl; 1109 u8 **ext_csd; 1110 u32 status; 1111 int ret; 1112 int i; 1113 1114 mq_rq = req_to_mmc_queue_req(req); 1115 rpmb_ioctl = (mq_rq->drv_op == MMC_DRV_OP_IOCTL_RPMB); 1116 1117 switch (mq_rq->drv_op) { 1118 case MMC_DRV_OP_IOCTL: 1119 if (card->ext_csd.cmdq_en) { 1120 ret = mmc_cmdq_disable(card); 1121 if (ret) 1122 break; 1123 } 1124 1125 mmc_blk_check_sbc(mq_rq); 1126 1127 fallthrough; 1128 case MMC_DRV_OP_IOCTL_RPMB: 1129 idata = mq_rq->drv_op_data; 1130 for (i = 0, ret = 0; i < mq_rq->ioc_count; i++) { 1131 ret = __mmc_blk_ioctl_cmd(card, md, idata, i); 1132 if (ret) 1133 break; 1134 } 1135 /* Always switch back to main area after RPMB access */ 1136 if (rpmb_ioctl) 1137 mmc_blk_part_switch(card, 0); 1138 else if (card->reenable_cmdq && !card->ext_csd.cmdq_en) 1139 mmc_cmdq_enable(card); 1140 break; 1141 case MMC_DRV_OP_BOOT_WP: 1142 ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BOOT_WP, 1143 card->ext_csd.boot_ro_lock | 1144 EXT_CSD_BOOT_WP_B_PWR_WP_EN, 1145 card->ext_csd.part_time); 1146 if (ret) 1147 pr_err("%s: Locking boot partition ro until next power on failed: %d\n", 1148 md->disk->disk_name, ret); 1149 else 1150 card->ext_csd.boot_ro_lock |= 1151 EXT_CSD_BOOT_WP_B_PWR_WP_EN; 1152 break; 1153 case MMC_DRV_OP_GET_CARD_STATUS: 1154 ret = mmc_send_status(card, &status); 1155 if (!ret) 1156 ret = status; 1157 break; 1158 case MMC_DRV_OP_GET_EXT_CSD: 1159 ext_csd = mq_rq->drv_op_data; 1160 ret = mmc_get_ext_csd(card, ext_csd); 1161 break; 1162 default: 1163 pr_err("%s: unknown driver specific operation\n", 1164 md->disk->disk_name); 1165 ret = -EINVAL; 1166 break; 1167 } 1168 mq_rq->drv_op_result = ret; 1169 blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK); 1170} 1171 1172static void mmc_blk_issue_erase_rq(struct mmc_queue *mq, struct request *req, 1173 int type, unsigned int erase_arg) 1174{ 1175 struct mmc_blk_data *md = mq->blkdata; 1176 struct mmc_card *card = md->queue.card; 1177 unsigned int nr; 1178 sector_t from; 1179 int err = 0; 1180 blk_status_t status = BLK_STS_OK; 1181 1182 if (!mmc_card_can_erase(card)) { 1183 status = BLK_STS_NOTSUPP; 1184 goto fail; 1185 } 1186 1187 from = blk_rq_pos(req); 1188 nr = blk_rq_sectors(req); 1189 1190 do { 1191 err = 0; 1192 if (card->quirks & MMC_QUIRK_INAND_CMD38) { 1193 err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, 1194 INAND_CMD38_ARG_EXT_CSD, 1195 erase_arg == MMC_TRIM_ARG ? 1196 INAND_CMD38_ARG_TRIM : 1197 INAND_CMD38_ARG_ERASE, 1198 card->ext_csd.generic_cmd6_time); 1199 } 1200 if (!err) 1201 err = mmc_erase(card, from, nr, erase_arg); 1202 } while (err == -EIO && !mmc_blk_reset(md, card->host, type)); 1203 if (err) 1204 status = BLK_STS_IOERR; 1205 else 1206 mmc_blk_reset_success(md, type); 1207fail: 1208 blk_mq_end_request(req, status); 1209} 1210 1211static void mmc_blk_issue_trim_rq(struct mmc_queue *mq, struct request *req) 1212{ 1213 mmc_blk_issue_erase_rq(mq, req, MMC_BLK_TRIM, MMC_TRIM_ARG); 1214} 1215 1216static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req) 1217{ 1218 struct mmc_blk_data *md = mq->blkdata; 1219 struct mmc_card *card = md->queue.card; 1220 unsigned int arg = card->erase_arg; 1221 1222 if (mmc_card_broken_sd_discard(card)) 1223 arg = SD_ERASE_ARG; 1224 1225 mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, arg); 1226} 1227 1228static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq, 1229 struct request *req) 1230{ 1231 struct mmc_blk_data *md = mq->blkdata; 1232 struct mmc_card *card = md->queue.card; 1233 unsigned int nr, arg; 1234 sector_t from; 1235 int err = 0, type = MMC_BLK_SECDISCARD; 1236 blk_status_t status = BLK_STS_OK; 1237 1238 if (!(mmc_card_can_secure_erase_trim(card))) { 1239 status = BLK_STS_NOTSUPP; 1240 goto out; 1241 } 1242 1243 from = blk_rq_pos(req); 1244 nr = blk_rq_sectors(req); 1245 1246 if (mmc_card_can_trim(card) && !mmc_erase_group_aligned(card, from, nr)) 1247 arg = MMC_SECURE_TRIM1_ARG; 1248 else 1249 arg = MMC_SECURE_ERASE_ARG; 1250 1251retry: 1252 if (card->quirks & MMC_QUIRK_INAND_CMD38) { 1253 err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, 1254 INAND_CMD38_ARG_EXT_CSD, 1255 arg == MMC_SECURE_TRIM1_ARG ? 1256 INAND_CMD38_ARG_SECTRIM1 : 1257 INAND_CMD38_ARG_SECERASE, 1258 card->ext_csd.generic_cmd6_time); 1259 if (err) 1260 goto out_retry; 1261 } 1262 1263 err = mmc_erase(card, from, nr, arg); 1264 if (err == -EIO) 1265 goto out_retry; 1266 if (err) { 1267 status = BLK_STS_IOERR; 1268 goto out; 1269 } 1270 1271 if (arg == MMC_SECURE_TRIM1_ARG) { 1272 if (card->quirks & MMC_QUIRK_INAND_CMD38) { 1273 err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, 1274 INAND_CMD38_ARG_EXT_CSD, 1275 INAND_CMD38_ARG_SECTRIM2, 1276 card->ext_csd.generic_cmd6_time); 1277 if (err) 1278 goto out_retry; 1279 } 1280 1281 err = mmc_erase(card, from, nr, MMC_SECURE_TRIM2_ARG); 1282 if (err == -EIO) 1283 goto out_retry; 1284 if (err) { 1285 status = BLK_STS_IOERR; 1286 goto out; 1287 } 1288 } 1289 1290out_retry: 1291 if (err && !mmc_blk_reset(md, card->host, type)) 1292 goto retry; 1293 if (!err) 1294 mmc_blk_reset_success(md, type); 1295out: 1296 blk_mq_end_request(req, status); 1297} 1298 1299static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req) 1300{ 1301 struct mmc_blk_data *md = mq->blkdata; 1302 struct mmc_card *card = md->queue.card; 1303 int ret = 0; 1304 1305 ret = mmc_flush_cache(card->host); 1306 blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK); 1307} 1308 1309/* 1310 * Reformat current write as a reliable write, supporting 1311 * both legacy and the enhanced reliable write MMC cards. 1312 * In each transfer we'll handle only as much as a single 1313 * reliable write can handle, thus finish the request in 1314 * partial completions. 1315 */ 1316static inline void mmc_apply_rel_rw(struct mmc_blk_request *brq, 1317 struct mmc_card *card, 1318 struct request *req) 1319{ 1320 if (!(card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN)) { 1321 /* Legacy mode imposes restrictions on transfers. */ 1322 if (!IS_ALIGNED(blk_rq_pos(req), card->ext_csd.rel_sectors)) 1323 brq->data.blocks = 1; 1324 1325 if (brq->data.blocks > card->ext_csd.rel_sectors) 1326 brq->data.blocks = card->ext_csd.rel_sectors; 1327 else if (brq->data.blocks < card->ext_csd.rel_sectors) 1328 brq->data.blocks = 1; 1329 } 1330} 1331 1332#define CMD_ERRORS_EXCL_OOR \ 1333 (R1_ADDRESS_ERROR | /* Misaligned address */ \ 1334 R1_BLOCK_LEN_ERROR | /* Transferred block length incorrect */\ 1335 R1_WP_VIOLATION | /* Tried to write to protected block */ \ 1336 R1_CARD_ECC_FAILED | /* Card ECC failed */ \ 1337 R1_CC_ERROR | /* Card controller error */ \ 1338 R1_ERROR) /* General/unknown error */ 1339 1340#define CMD_ERRORS \ 1341 (CMD_ERRORS_EXCL_OOR | \ 1342 R1_OUT_OF_RANGE) /* Command argument out of range */ \ 1343 1344static void mmc_blk_eval_resp_error(struct mmc_blk_request *brq) 1345{ 1346 u32 val; 1347 1348 /* 1349 * Per the SD specification(physical layer version 4.10)[1], 1350 * section 4.3.3, it explicitly states that "When the last 1351 * block of user area is read using CMD18, the host should 1352 * ignore OUT_OF_RANGE error that may occur even the sequence 1353 * is correct". And JESD84-B51 for eMMC also has a similar 1354 * statement on section 6.8.3. 1355 * 1356 * Multiple block read/write could be done by either predefined 1357 * method, namely CMD23, or open-ending mode. For open-ending mode, 1358 * we should ignore the OUT_OF_RANGE error as it's normal behaviour. 1359 * 1360 * However the spec[1] doesn't tell us whether we should also 1361 * ignore that for predefined method. But per the spec[1], section 1362 * 4.15 Set Block Count Command, it says"If illegal block count 1363 * is set, out of range error will be indicated during read/write 1364 * operation (For example, data transfer is stopped at user area 1365 * boundary)." In another word, we could expect a out of range error 1366 * in the response for the following CMD18/25. And if argument of 1367 * CMD23 + the argument of CMD18/25 exceed the max number of blocks, 1368 * we could also expect to get a -ETIMEDOUT or any error number from 1369 * the host drivers due to missing data response(for write)/data(for 1370 * read), as the cards will stop the data transfer by itself per the 1371 * spec. So we only need to check R1_OUT_OF_RANGE for open-ending mode. 1372 */ 1373 1374 if (!brq->stop.error) { 1375 bool oor_with_open_end; 1376 /* If there is no error yet, check R1 response */ 1377 1378 val = brq->stop.resp[0] & CMD_ERRORS; 1379 oor_with_open_end = val & R1_OUT_OF_RANGE && !brq->mrq.sbc; 1380 1381 if (val && !oor_with_open_end) 1382 brq->stop.error = -EIO; 1383 } 1384} 1385 1386static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq, 1387 int recovery_mode, bool *do_rel_wr_p, 1388 bool *do_data_tag_p) 1389{ 1390 struct mmc_blk_data *md = mq->blkdata; 1391 struct mmc_card *card = md->queue.card; 1392 struct mmc_blk_request *brq = &mqrq->brq; 1393 struct request *req = mmc_queue_req_to_req(mqrq); 1394 bool do_rel_wr, do_data_tag; 1395 1396 /* 1397 * Reliable writes are used to implement Forced Unit Access and 1398 * are supported only on MMCs. 1399 */ 1400 do_rel_wr = (req->cmd_flags & REQ_FUA) && 1401 rq_data_dir(req) == WRITE && 1402 (md->flags & MMC_BLK_REL_WR); 1403 1404 memset(brq, 0, sizeof(struct mmc_blk_request)); 1405 1406 mmc_crypto_prepare_req(mqrq); 1407 1408 brq->mrq.data = &brq->data; 1409 brq->mrq.tag = req->tag; 1410 1411 brq->stop.opcode = MMC_STOP_TRANSMISSION; 1412 brq->stop.arg = 0; 1413 1414 if (rq_data_dir(req) == READ) { 1415 brq->data.flags = MMC_DATA_READ; 1416 brq->stop.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC; 1417 } else { 1418 brq->data.flags = MMC_DATA_WRITE; 1419 brq->stop.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC; 1420 } 1421 1422 brq->data.blksz = 512; 1423 brq->data.blocks = blk_rq_sectors(req); 1424 brq->data.blk_addr = blk_rq_pos(req); 1425 1426 /* 1427 * The command queue supports 2 priorities: "high" (1) and "simple" (0). 1428 * The eMMC will give "high" priority tasks priority over "simple" 1429 * priority tasks. Here we always set "simple" priority by not setting 1430 * MMC_DATA_PRIO. 1431 */ 1432 1433 /* 1434 * The block layer doesn't support all sector count 1435 * restrictions, so we need to be prepared for too big 1436 * requests. 1437 */ 1438 if (brq->data.blocks > card->host->max_blk_count) 1439 brq->data.blocks = card->host->max_blk_count; 1440 1441 if (brq->data.blocks > 1) { 1442 /* 1443 * Some SD cards in SPI mode return a CRC error or even lock up 1444 * completely when trying to read the last block using a 1445 * multiblock read command. 1446 */ 1447 if (mmc_host_is_spi(card->host) && (rq_data_dir(req) == READ) && 1448 (blk_rq_pos(req) + blk_rq_sectors(req) == 1449 get_capacity(md->disk))) 1450 brq->data.blocks--; 1451 1452 /* 1453 * After a read error, we redo the request one (native) sector 1454 * at a time in order to accurately determine which 1455 * sectors can be read successfully. 1456 */ 1457 if (recovery_mode) 1458 brq->data.blocks = queue_physical_block_size(mq->queue) >> 9; 1459 1460 /* 1461 * Some controllers have HW issues while operating 1462 * in multiple I/O mode 1463 */ 1464 if (card->host->ops->multi_io_quirk) 1465 brq->data.blocks = card->host->ops->multi_io_quirk(card, 1466 (rq_data_dir(req) == READ) ? 1467 MMC_DATA_READ : MMC_DATA_WRITE, 1468 brq->data.blocks); 1469 } 1470 1471 if (do_rel_wr) { 1472 mmc_apply_rel_rw(brq, card, req); 1473 brq->data.flags |= MMC_DATA_REL_WR; 1474 } 1475 1476 /* 1477 * Data tag is used only during writing meta data to speed 1478 * up write and any subsequent read of this meta data 1479 */ 1480 do_data_tag = card->ext_csd.data_tag_unit_size && 1481 (req->cmd_flags & REQ_META) && 1482 (rq_data_dir(req) == WRITE) && 1483 ((brq->data.blocks * brq->data.blksz) >= 1484 card->ext_csd.data_tag_unit_size); 1485 1486 if (do_data_tag) 1487 brq->data.flags |= MMC_DATA_DAT_TAG; 1488 1489 mmc_set_data_timeout(&brq->data, card); 1490 1491 brq->data.sg = mqrq->sg; 1492 brq->data.sg_len = mmc_queue_map_sg(mq, mqrq); 1493 1494 /* 1495 * Adjust the sg list so it is the same size as the 1496 * request. 1497 */ 1498 if (brq->data.blocks != blk_rq_sectors(req)) { 1499 int i, data_size = brq->data.blocks << 9; 1500 struct scatterlist *sg; 1501 1502 for_each_sg(brq->data.sg, sg, brq->data.sg_len, i) { 1503 data_size -= sg->length; 1504 if (data_size <= 0) { 1505 sg->length += data_size; 1506 i++; 1507 break; 1508 } 1509 } 1510 brq->data.sg_len = i; 1511 } 1512 1513 if (do_rel_wr_p) 1514 *do_rel_wr_p = do_rel_wr; 1515 1516 if (do_data_tag_p) 1517 *do_data_tag_p = do_data_tag; 1518} 1519 1520#define MMC_CQE_RETRIES 2 1521 1522static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req) 1523{ 1524 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1525 struct mmc_request *mrq = &mqrq->brq.mrq; 1526 struct request_queue *q = req->q; 1527 struct mmc_host *host = mq->card->host; 1528 enum mmc_issue_type issue_type = mmc_issue_type(mq, req); 1529 unsigned long flags; 1530 bool put_card; 1531 int err; 1532 1533 mmc_cqe_post_req(host, mrq); 1534 1535 if (mrq->cmd && mrq->cmd->error) 1536 err = mrq->cmd->error; 1537 else if (mrq->data && mrq->data->error) 1538 err = mrq->data->error; 1539 else 1540 err = 0; 1541 1542 if (err) { 1543 if (mqrq->retries++ < MMC_CQE_RETRIES) 1544 blk_mq_requeue_request(req, true); 1545 else 1546 blk_mq_end_request(req, BLK_STS_IOERR); 1547 } else if (mrq->data) { 1548 if (blk_update_request(req, BLK_STS_OK, mrq->data->bytes_xfered)) 1549 blk_mq_requeue_request(req, true); 1550 else 1551 __blk_mq_end_request(req, BLK_STS_OK); 1552 } else if (mq->in_recovery) { 1553 blk_mq_requeue_request(req, true); 1554 } else { 1555 blk_mq_end_request(req, BLK_STS_OK); 1556 } 1557 1558 spin_lock_irqsave(&mq->lock, flags); 1559 1560 mq->in_flight[issue_type] -= 1; 1561 1562 put_card = (mmc_tot_in_flight(mq) == 0); 1563 1564 mmc_cqe_check_busy(mq); 1565 1566 spin_unlock_irqrestore(&mq->lock, flags); 1567 1568 if (!mq->cqe_busy) 1569 blk_mq_run_hw_queues(q, true); 1570 1571 if (put_card) 1572 mmc_put_card(mq->card, &mq->ctx); 1573} 1574 1575void mmc_blk_cqe_recovery(struct mmc_queue *mq) 1576{ 1577 struct mmc_card *card = mq->card; 1578 struct mmc_host *host = card->host; 1579 int err; 1580 1581 pr_debug("%s: CQE recovery start\n", mmc_hostname(host)); 1582 1583 err = mmc_cqe_recovery(host); 1584 if (err) 1585 mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY); 1586 mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY); 1587 1588 pr_debug("%s: CQE recovery done\n", mmc_hostname(host)); 1589} 1590 1591static void mmc_blk_cqe_req_done(struct mmc_request *mrq) 1592{ 1593 struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req, 1594 brq.mrq); 1595 struct request *req = mmc_queue_req_to_req(mqrq); 1596 struct request_queue *q = req->q; 1597 struct mmc_queue *mq = q->queuedata; 1598 1599 /* 1600 * Block layer timeouts race with completions which means the normal 1601 * completion path cannot be used during recovery. 1602 */ 1603 if (mq->in_recovery) 1604 mmc_blk_cqe_complete_rq(mq, req); 1605 else if (likely(!blk_should_fake_timeout(req->q))) 1606 blk_mq_complete_request(req); 1607} 1608 1609static int mmc_blk_cqe_start_req(struct mmc_host *host, struct mmc_request *mrq) 1610{ 1611 mrq->done = mmc_blk_cqe_req_done; 1612 mrq->recovery_notifier = mmc_cqe_recovery_notifier; 1613 1614 return mmc_cqe_start_req(host, mrq); 1615} 1616 1617static struct mmc_request *mmc_blk_cqe_prep_dcmd(struct mmc_queue_req *mqrq, 1618 struct request *req) 1619{ 1620 struct mmc_blk_request *brq = &mqrq->brq; 1621 1622 memset(brq, 0, sizeof(*brq)); 1623 1624 brq->mrq.cmd = &brq->cmd; 1625 brq->mrq.tag = req->tag; 1626 1627 return &brq->mrq; 1628} 1629 1630static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request *req) 1631{ 1632 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1633 struct mmc_request *mrq = mmc_blk_cqe_prep_dcmd(mqrq, req); 1634 1635 mrq->cmd->opcode = MMC_SWITCH; 1636 mrq->cmd->arg = (MMC_SWITCH_MODE_WRITE_BYTE << 24) | 1637 (EXT_CSD_FLUSH_CACHE << 16) | 1638 (1 << 8) | 1639 EXT_CSD_CMD_SET_NORMAL; 1640 mrq->cmd->flags = MMC_CMD_AC | MMC_RSP_R1B; 1641 1642 return mmc_blk_cqe_start_req(mq->card->host, mrq); 1643} 1644 1645static int mmc_blk_hsq_issue_rw_rq(struct mmc_queue *mq, struct request *req) 1646{ 1647 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1648 struct mmc_host *host = mq->card->host; 1649 int err; 1650 1651 mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq); 1652 mqrq->brq.mrq.done = mmc_blk_hsq_req_done; 1653 mmc_pre_req(host, &mqrq->brq.mrq); 1654 1655 err = mmc_cqe_start_req(host, &mqrq->brq.mrq); 1656 if (err) 1657 mmc_post_req(host, &mqrq->brq.mrq, err); 1658 1659 return err; 1660} 1661 1662static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request *req) 1663{ 1664 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1665 struct mmc_host *host = mq->card->host; 1666 1667 if (host->hsq_enabled) 1668 return mmc_blk_hsq_issue_rw_rq(mq, req); 1669 1670 mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL); 1671 1672 return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq); 1673} 1674 1675static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq, 1676 struct mmc_card *card, 1677 int recovery_mode, 1678 struct mmc_queue *mq) 1679{ 1680 u32 readcmd, writecmd; 1681 struct mmc_blk_request *brq = &mqrq->brq; 1682 struct request *req = mmc_queue_req_to_req(mqrq); 1683 struct mmc_blk_data *md = mq->blkdata; 1684 bool do_rel_wr, do_data_tag; 1685 1686 mmc_blk_data_prep(mq, mqrq, recovery_mode, &do_rel_wr, &do_data_tag); 1687 1688 brq->mrq.cmd = &brq->cmd; 1689 1690 brq->cmd.arg = blk_rq_pos(req); 1691 if (!mmc_card_blockaddr(card)) 1692 brq->cmd.arg <<= 9; 1693 brq->cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC; 1694 1695 if (brq->data.blocks > 1 || do_rel_wr) { 1696 /* SPI multiblock writes terminate using a special 1697 * token, not a STOP_TRANSMISSION request. 1698 */ 1699 if (!mmc_host_is_spi(card->host) || 1700 rq_data_dir(req) == READ) 1701 brq->mrq.stop = &brq->stop; 1702 readcmd = MMC_READ_MULTIPLE_BLOCK; 1703 writecmd = MMC_WRITE_MULTIPLE_BLOCK; 1704 } else { 1705 brq->mrq.stop = NULL; 1706 readcmd = MMC_READ_SINGLE_BLOCK; 1707 writecmd = MMC_WRITE_BLOCK; 1708 } 1709 brq->cmd.opcode = rq_data_dir(req) == READ ? readcmd : writecmd; 1710 1711 /* 1712 * Pre-defined multi-block transfers are preferable to 1713 * open ended-ones (and necessary for reliable writes). 1714 * However, it is not sufficient to just send CMD23, 1715 * and avoid the final CMD12, as on an error condition 1716 * CMD12 (stop) needs to be sent anyway. This, coupled 1717 * with Auto-CMD23 enhancements provided by some 1718 * hosts, means that the complexity of dealing 1719 * with this is best left to the host. If CMD23 is 1720 * supported by card and host, we'll fill sbc in and let 1721 * the host deal with handling it correctly. This means 1722 * that for hosts that don't expose MMC_CAP_CMD23, no 1723 * change of behavior will be observed. 1724 * 1725 * N.B: Some MMC cards experience perf degradation. 1726 * We'll avoid using CMD23-bounded multiblock writes for 1727 * these, while retaining features like reliable writes. 1728 */ 1729 if ((md->flags & MMC_BLK_CMD23) && mmc_op_multi(brq->cmd.opcode) && 1730 (do_rel_wr || !mmc_card_blk_no_cmd23(card) || do_data_tag)) { 1731 brq->sbc.opcode = MMC_SET_BLOCK_COUNT; 1732 brq->sbc.arg = brq->data.blocks | 1733 (do_rel_wr ? (1 << 31) : 0) | 1734 (do_data_tag ? (1 << 29) : 0); 1735 brq->sbc.flags = MMC_RSP_R1 | MMC_CMD_AC; 1736 brq->mrq.sbc = &brq->sbc; 1737 } 1738 1739 if (mmc_card_ult_capacity(card)) { 1740 brq->cmd.ext_addr = blk_rq_pos(req) >> 32; 1741 brq->cmd.has_ext_addr = true; 1742 } 1743} 1744 1745#define MMC_MAX_RETRIES 5 1746#define MMC_DATA_RETRIES 2 1747#define MMC_NO_RETRIES (MMC_MAX_RETRIES + 1) 1748 1749static int mmc_blk_send_stop(struct mmc_card *card, unsigned int timeout) 1750{ 1751 struct mmc_command cmd = { 1752 .opcode = MMC_STOP_TRANSMISSION, 1753 .flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC, 1754 /* Some hosts wait for busy anyway, so provide a busy timeout */ 1755 .busy_timeout = timeout, 1756 }; 1757 1758 return mmc_wait_for_cmd(card->host, &cmd, 5); 1759} 1760 1761static int mmc_blk_fix_state(struct mmc_card *card, struct request *req) 1762{ 1763 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1764 struct mmc_blk_request *brq = &mqrq->brq; 1765 unsigned int timeout = mmc_blk_data_timeout_ms(card->host, &brq->data); 1766 int err; 1767 1768 mmc_retune_hold_now(card->host); 1769 1770 mmc_blk_send_stop(card, timeout); 1771 1772 err = mmc_poll_for_busy(card, timeout, false, MMC_BUSY_IO); 1773 1774 mmc_retune_release(card->host); 1775 1776 return err; 1777} 1778 1779#define MMC_READ_SINGLE_RETRIES 2 1780 1781/* Single (native) sector read during recovery */ 1782static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req) 1783{ 1784 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1785 struct mmc_request *mrq = &mqrq->brq.mrq; 1786 struct mmc_card *card = mq->card; 1787 struct mmc_host *host = card->host; 1788 blk_status_t error = BLK_STS_OK; 1789 size_t bytes_per_read = queue_physical_block_size(mq->queue); 1790 1791 do { 1792 u32 status; 1793 int err; 1794 int retries = 0; 1795 1796 while (retries++ <= MMC_READ_SINGLE_RETRIES) { 1797 mmc_blk_rw_rq_prep(mqrq, card, 1, mq); 1798 1799 mmc_wait_for_req(host, mrq); 1800 1801 err = mmc_send_status(card, &status); 1802 if (err) 1803 goto error_exit; 1804 1805 if (!mmc_host_is_spi(host) && 1806 !mmc_ready_for_data(status)) { 1807 err = mmc_blk_fix_state(card, req); 1808 if (err) 1809 goto error_exit; 1810 } 1811 1812 if (!mrq->cmd->error) 1813 break; 1814 } 1815 1816 if (mrq->cmd->error || 1817 mrq->data->error || 1818 (!mmc_host_is_spi(host) && 1819 (mrq->cmd->resp[0] & CMD_ERRORS || status & CMD_ERRORS))) 1820 error = BLK_STS_IOERR; 1821 else 1822 error = BLK_STS_OK; 1823 1824 } while (blk_update_request(req, error, bytes_per_read)); 1825 1826 return; 1827 1828error_exit: 1829 mrq->data->bytes_xfered = 0; 1830 blk_update_request(req, BLK_STS_IOERR, bytes_per_read); 1831 /* Let it try the remaining request again */ 1832 if (mqrq->retries > MMC_MAX_RETRIES - 1) 1833 mqrq->retries = MMC_MAX_RETRIES - 1; 1834} 1835 1836static inline bool mmc_blk_oor_valid(struct mmc_blk_request *brq) 1837{ 1838 return !!brq->mrq.sbc; 1839} 1840 1841static inline u32 mmc_blk_stop_err_bits(struct mmc_blk_request *brq) 1842{ 1843 return mmc_blk_oor_valid(brq) ? CMD_ERRORS : CMD_ERRORS_EXCL_OOR; 1844} 1845 1846/* 1847 * Check for errors the host controller driver might not have seen such as 1848 * response mode errors or invalid card state. 1849 */ 1850static bool mmc_blk_status_error(struct request *req, u32 status) 1851{ 1852 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1853 struct mmc_blk_request *brq = &mqrq->brq; 1854 struct mmc_queue *mq = req->q->queuedata; 1855 u32 stop_err_bits; 1856 1857 if (mmc_host_is_spi(mq->card->host)) 1858 return false; 1859 1860 stop_err_bits = mmc_blk_stop_err_bits(brq); 1861 1862 return brq->cmd.resp[0] & CMD_ERRORS || 1863 brq->stop.resp[0] & stop_err_bits || 1864 status & stop_err_bits || 1865 (rq_data_dir(req) == WRITE && !mmc_ready_for_data(status)); 1866} 1867 1868static inline bool mmc_blk_cmd_started(struct mmc_blk_request *brq) 1869{ 1870 return !brq->sbc.error && !brq->cmd.error && 1871 !(brq->cmd.resp[0] & CMD_ERRORS); 1872} 1873 1874/* 1875 * Requests are completed by mmc_blk_mq_complete_rq() which sets simple 1876 * policy: 1877 * 1. A request that has transferred at least some data is considered 1878 * successful and will be requeued if there is remaining data to 1879 * transfer. 1880 * 2. Otherwise the number of retries is incremented and the request 1881 * will be requeued if there are remaining retries. 1882 * 3. Otherwise the request will be errored out. 1883 * That means mmc_blk_mq_complete_rq() is controlled by bytes_xfered and 1884 * mqrq->retries. So there are only 4 possible actions here: 1885 * 1. do not accept the bytes_xfered value i.e. set it to zero 1886 * 2. change mqrq->retries to determine the number of retries 1887 * 3. try to reset the card 1888 * 4. read one sector at a time 1889 */ 1890static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req) 1891{ 1892 int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE; 1893 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1894 struct mmc_blk_request *brq = &mqrq->brq; 1895 struct mmc_blk_data *md = mq->blkdata; 1896 struct mmc_card *card = mq->card; 1897 u32 status; 1898 u32 blocks; 1899 int err; 1900 1901 /* 1902 * Some errors the host driver might not have seen. Set the number of 1903 * bytes transferred to zero in that case. 1904 */ 1905 err = __mmc_send_status(card, &status, 0); 1906 if (err || mmc_blk_status_error(req, status)) 1907 brq->data.bytes_xfered = 0; 1908 1909 mmc_retune_release(card->host); 1910 1911 /* 1912 * Try again to get the status. This also provides an opportunity for 1913 * re-tuning. 1914 */ 1915 if (err) 1916 err = __mmc_send_status(card, &status, 0); 1917 1918 /* 1919 * Nothing more to do after the number of bytes transferred has been 1920 * updated and there is no card. 1921 */ 1922 if (err && mmc_detect_card_removed(card->host)) 1923 return; 1924 1925 /* Try to get back to "tran" state */ 1926 if (!mmc_host_is_spi(mq->card->host) && 1927 (err || !mmc_ready_for_data(status))) 1928 err = mmc_blk_fix_state(mq->card, req); 1929 1930 /* 1931 * Special case for SD cards where the card might record the number of 1932 * blocks written. 1933 */ 1934 if (!err && mmc_blk_cmd_started(brq) && mmc_card_sd(card) && 1935 rq_data_dir(req) == WRITE) { 1936 if (mmc_sd_num_wr_blocks(card, &blocks)) 1937 brq->data.bytes_xfered = 0; 1938 else 1939 brq->data.bytes_xfered = blocks << 9; 1940 } 1941 1942 /* Reset if the card is in a bad state */ 1943 if (!mmc_host_is_spi(mq->card->host) && 1944 err && mmc_blk_reset(md, card->host, type)) { 1945 pr_err("%s: recovery failed!\n", req->q->disk->disk_name); 1946 mqrq->retries = MMC_NO_RETRIES; 1947 return; 1948 } 1949 1950 /* 1951 * If anything was done, just return and if there is anything remaining 1952 * on the request it will get requeued. 1953 */ 1954 if (brq->data.bytes_xfered) 1955 return; 1956 1957 /* Reset before last retry */ 1958 if (mqrq->retries + 1 == MMC_MAX_RETRIES && 1959 mmc_blk_reset(md, card->host, type)) 1960 return; 1961 1962 /* Command errors fail fast, so use all MMC_MAX_RETRIES */ 1963 if (brq->sbc.error || brq->cmd.error) 1964 return; 1965 1966 /* Reduce the remaining retries for data errors */ 1967 if (mqrq->retries < MMC_MAX_RETRIES - MMC_DATA_RETRIES) { 1968 mqrq->retries = MMC_MAX_RETRIES - MMC_DATA_RETRIES; 1969 return; 1970 } 1971 1972 if (rq_data_dir(req) == READ && brq->data.blocks > 1973 queue_physical_block_size(mq->queue) >> 9) { 1974 /* Read one (native) sector at a time */ 1975 mmc_blk_read_single(mq, req); 1976 return; 1977 } 1978} 1979 1980static inline bool mmc_blk_rq_error(struct mmc_blk_request *brq) 1981{ 1982 mmc_blk_eval_resp_error(brq); 1983 1984 return brq->sbc.error || brq->cmd.error || brq->stop.error || 1985 brq->data.error || brq->cmd.resp[0] & CMD_ERRORS; 1986} 1987 1988static int mmc_spi_err_check(struct mmc_card *card) 1989{ 1990 u32 status = 0; 1991 int err; 1992 1993 /* 1994 * SPI does not have a TRAN state we have to wait on, instead the 1995 * card is ready again when it no longer holds the line LOW. 1996 * We still have to ensure two things here before we know the write 1997 * was successful: 1998 * 1. The card has not disconnected during busy and we actually read our 1999 * own pull-up, thinking it was still connected, so ensure it 2000 * still responds. 2001 * 2. Check for any error bits, in particular R1_SPI_IDLE to catch a 2002 * just reconnected card after being disconnected during busy. 2003 */ 2004 err = __mmc_send_status(card, &status, 0); 2005 if (err) 2006 return err; 2007 /* All R1 and R2 bits of SPI are errors in our case */ 2008 if (status) 2009 return -EIO; 2010 return 0; 2011} 2012 2013static int mmc_blk_busy_cb(void *cb_data, bool *busy) 2014{ 2015 struct mmc_blk_busy_data *data = cb_data; 2016 u32 status = 0; 2017 int err; 2018 2019 err = mmc_send_status(data->card, &status); 2020 if (err) 2021 return err; 2022 2023 /* Accumulate response error bits. */ 2024 data->status |= status; 2025 2026 *busy = !mmc_ready_for_data(status); 2027 return 0; 2028} 2029 2030static int mmc_blk_card_busy(struct mmc_card *card, struct request *req) 2031{ 2032 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2033 struct mmc_blk_busy_data cb_data; 2034 int err; 2035 2036 if (rq_data_dir(req) == READ) 2037 return 0; 2038 2039 if (mmc_host_is_spi(card->host)) { 2040 err = mmc_spi_err_check(card); 2041 if (err) 2042 mqrq->brq.data.bytes_xfered = 0; 2043 return err; 2044 } 2045 2046 cb_data.card = card; 2047 cb_data.status = 0; 2048 err = __mmc_poll_for_busy(card->host, 0, MMC_BLK_TIMEOUT_MS, 2049 &mmc_blk_busy_cb, &cb_data); 2050 2051 /* 2052 * Do not assume data transferred correctly if there are any error bits 2053 * set. 2054 */ 2055 if (cb_data.status & mmc_blk_stop_err_bits(&mqrq->brq)) { 2056 mqrq->brq.data.bytes_xfered = 0; 2057 err = err ? err : -EIO; 2058 } 2059 2060 /* Copy the exception bit so it will be seen later on */ 2061 if (mmc_card_mmc(card) && cb_data.status & R1_EXCEPTION_EVENT) 2062 mqrq->brq.cmd.resp[0] |= R1_EXCEPTION_EVENT; 2063 2064 return err; 2065} 2066 2067static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq, 2068 struct request *req) 2069{ 2070 int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE; 2071 2072 mmc_blk_reset_success(mq->blkdata, type); 2073} 2074 2075static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req) 2076{ 2077 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2078 unsigned int nr_bytes = mqrq->brq.data.bytes_xfered; 2079 2080 if (nr_bytes) { 2081 if (blk_update_request(req, BLK_STS_OK, nr_bytes)) 2082 blk_mq_requeue_request(req, true); 2083 else 2084 __blk_mq_end_request(req, BLK_STS_OK); 2085 } else if (!blk_rq_bytes(req)) { 2086 __blk_mq_end_request(req, BLK_STS_IOERR); 2087 } else if (mqrq->retries++ < MMC_MAX_RETRIES) { 2088 blk_mq_requeue_request(req, true); 2089 } else { 2090 if (mmc_card_removed(mq->card)) 2091 req->rq_flags |= RQF_QUIET; 2092 blk_mq_end_request(req, BLK_STS_IOERR); 2093 } 2094} 2095 2096static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq, 2097 struct mmc_queue_req *mqrq) 2098{ 2099 return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) && 2100 (mqrq->brq.cmd.resp[0] & R1_EXCEPTION_EVENT || 2101 mqrq->brq.stop.resp[0] & R1_EXCEPTION_EVENT); 2102} 2103 2104static void mmc_blk_urgent_bkops(struct mmc_queue *mq, 2105 struct mmc_queue_req *mqrq) 2106{ 2107 if (mmc_blk_urgent_bkops_needed(mq, mqrq)) 2108 mmc_run_bkops(mq->card); 2109} 2110 2111static void mmc_blk_hsq_req_done(struct mmc_request *mrq) 2112{ 2113 struct mmc_queue_req *mqrq = 2114 container_of(mrq, struct mmc_queue_req, brq.mrq); 2115 struct request *req = mmc_queue_req_to_req(mqrq); 2116 struct request_queue *q = req->q; 2117 struct mmc_queue *mq = q->queuedata; 2118 struct mmc_host *host = mq->card->host; 2119 unsigned long flags; 2120 2121 if (mmc_blk_rq_error(&mqrq->brq) || 2122 mmc_blk_urgent_bkops_needed(mq, mqrq)) { 2123 spin_lock_irqsave(&mq->lock, flags); 2124 mq->recovery_needed = true; 2125 mq->recovery_req = req; 2126 spin_unlock_irqrestore(&mq->lock, flags); 2127 2128 host->cqe_ops->cqe_recovery_start(host); 2129 2130 schedule_work(&mq->recovery_work); 2131 return; 2132 } 2133 2134 mmc_blk_rw_reset_success(mq, req); 2135 2136 /* 2137 * Block layer timeouts race with completions which means the normal 2138 * completion path cannot be used during recovery. 2139 */ 2140 if (mq->in_recovery) 2141 mmc_blk_cqe_complete_rq(mq, req); 2142 else if (likely(!blk_should_fake_timeout(req->q))) 2143 blk_mq_complete_request(req); 2144} 2145 2146void mmc_blk_mq_complete(struct request *req) 2147{ 2148 struct mmc_queue *mq = req->q->queuedata; 2149 struct mmc_host *host = mq->card->host; 2150 2151 if (host->cqe_enabled) 2152 mmc_blk_cqe_complete_rq(mq, req); 2153 else if (likely(!blk_should_fake_timeout(req->q))) 2154 mmc_blk_mq_complete_rq(mq, req); 2155} 2156 2157static void mmc_blk_mq_poll_completion(struct mmc_queue *mq, 2158 struct request *req) 2159{ 2160 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2161 struct mmc_host *host = mq->card->host; 2162 2163 if (mmc_blk_rq_error(&mqrq->brq) || 2164 mmc_blk_card_busy(mq->card, req)) { 2165 mmc_blk_mq_rw_recovery(mq, req); 2166 } else { 2167 mmc_blk_rw_reset_success(mq, req); 2168 mmc_retune_release(host); 2169 } 2170 2171 mmc_blk_urgent_bkops(mq, mqrq); 2172} 2173 2174static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, enum mmc_issue_type issue_type) 2175{ 2176 unsigned long flags; 2177 bool put_card; 2178 2179 spin_lock_irqsave(&mq->lock, flags); 2180 2181 mq->in_flight[issue_type] -= 1; 2182 2183 put_card = (mmc_tot_in_flight(mq) == 0); 2184 2185 spin_unlock_irqrestore(&mq->lock, flags); 2186 2187 if (put_card) 2188 mmc_put_card(mq->card, &mq->ctx); 2189} 2190 2191static void mmc_blk_mq_post_req(struct mmc_queue *mq, struct request *req, 2192 bool can_sleep) 2193{ 2194 enum mmc_issue_type issue_type = mmc_issue_type(mq, req); 2195 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2196 struct mmc_request *mrq = &mqrq->brq.mrq; 2197 struct mmc_host *host = mq->card->host; 2198 2199 mmc_post_req(host, mrq, 0); 2200 2201 /* 2202 * Block layer timeouts race with completions which means the normal 2203 * completion path cannot be used during recovery. 2204 */ 2205 if (mq->in_recovery) { 2206 mmc_blk_mq_complete_rq(mq, req); 2207 } else if (likely(!blk_should_fake_timeout(req->q))) { 2208 if (can_sleep) 2209 blk_mq_complete_request_direct(req, mmc_blk_mq_complete); 2210 else 2211 blk_mq_complete_request(req); 2212 } 2213 2214 mmc_blk_mq_dec_in_flight(mq, issue_type); 2215} 2216 2217void mmc_blk_mq_recovery(struct mmc_queue *mq) 2218{ 2219 struct request *req = mq->recovery_req; 2220 struct mmc_host *host = mq->card->host; 2221 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2222 2223 mq->recovery_req = NULL; 2224 mq->rw_wait = false; 2225 2226 if (mmc_blk_rq_error(&mqrq->brq)) { 2227 mmc_retune_hold_now(host); 2228 mmc_blk_mq_rw_recovery(mq, req); 2229 } 2230 2231 mmc_blk_urgent_bkops(mq, mqrq); 2232 2233 mmc_blk_mq_post_req(mq, req, true); 2234} 2235 2236static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq, 2237 struct request **prev_req) 2238{ 2239 if (mmc_host_can_done_complete(mq->card->host)) 2240 return; 2241 2242 mutex_lock(&mq->complete_lock); 2243 2244 if (!mq->complete_req) 2245 goto out_unlock; 2246 2247 mmc_blk_mq_poll_completion(mq, mq->complete_req); 2248 2249 if (prev_req) 2250 *prev_req = mq->complete_req; 2251 else 2252 mmc_blk_mq_post_req(mq, mq->complete_req, true); 2253 2254 mq->complete_req = NULL; 2255 2256out_unlock: 2257 mutex_unlock(&mq->complete_lock); 2258} 2259 2260void mmc_blk_mq_complete_work(struct work_struct *work) 2261{ 2262 struct mmc_queue *mq = container_of(work, struct mmc_queue, 2263 complete_work); 2264 2265 mmc_blk_mq_complete_prev_req(mq, NULL); 2266} 2267 2268static void mmc_blk_mq_req_done(struct mmc_request *mrq) 2269{ 2270 struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req, 2271 brq.mrq); 2272 struct request *req = mmc_queue_req_to_req(mqrq); 2273 struct request_queue *q = req->q; 2274 struct mmc_queue *mq = q->queuedata; 2275 struct mmc_host *host = mq->card->host; 2276 unsigned long flags; 2277 2278 if (!mmc_host_can_done_complete(host)) { 2279 bool waiting; 2280 2281 /* 2282 * We cannot complete the request in this context, so record 2283 * that there is a request to complete, and that a following 2284 * request does not need to wait (although it does need to 2285 * complete complete_req first). 2286 */ 2287 spin_lock_irqsave(&mq->lock, flags); 2288 mq->complete_req = req; 2289 mq->rw_wait = false; 2290 waiting = mq->waiting; 2291 spin_unlock_irqrestore(&mq->lock, flags); 2292 2293 /* 2294 * If 'waiting' then the waiting task will complete this 2295 * request, otherwise queue a work to do it. Note that 2296 * complete_work may still race with the dispatch of a following 2297 * request. 2298 */ 2299 if (waiting) 2300 wake_up(&mq->wait); 2301 else 2302 queue_work(mq->card->complete_wq, &mq->complete_work); 2303 2304 return; 2305 } 2306 2307 /* Take the recovery path for errors or urgent background operations */ 2308 if (mmc_blk_rq_error(&mqrq->brq) || 2309 mmc_blk_urgent_bkops_needed(mq, mqrq)) { 2310 spin_lock_irqsave(&mq->lock, flags); 2311 mq->recovery_needed = true; 2312 mq->recovery_req = req; 2313 spin_unlock_irqrestore(&mq->lock, flags); 2314 wake_up(&mq->wait); 2315 schedule_work(&mq->recovery_work); 2316 return; 2317 } 2318 2319 mmc_blk_rw_reset_success(mq, req); 2320 2321 mq->rw_wait = false; 2322 wake_up(&mq->wait); 2323 2324 /* context unknown */ 2325 mmc_blk_mq_post_req(mq, req, false); 2326} 2327 2328static bool mmc_blk_rw_wait_cond(struct mmc_queue *mq, int *err) 2329{ 2330 unsigned long flags; 2331 bool done; 2332 2333 /* 2334 * Wait while there is another request in progress, but not if recovery 2335 * is needed. Also indicate whether there is a request waiting to start. 2336 */ 2337 spin_lock_irqsave(&mq->lock, flags); 2338 if (mq->recovery_needed) { 2339 *err = -EBUSY; 2340 done = true; 2341 } else { 2342 done = !mq->rw_wait; 2343 } 2344 mq->waiting = !done; 2345 spin_unlock_irqrestore(&mq->lock, flags); 2346 2347 return done; 2348} 2349 2350static int mmc_blk_rw_wait(struct mmc_queue *mq, struct request **prev_req) 2351{ 2352 int err = 0; 2353 2354 wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err)); 2355 2356 /* Always complete the previous request if there is one */ 2357 mmc_blk_mq_complete_prev_req(mq, prev_req); 2358 2359 return err; 2360} 2361 2362static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq, 2363 struct request *req) 2364{ 2365 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2366 struct mmc_host *host = mq->card->host; 2367 struct request *prev_req = NULL; 2368 int err = 0; 2369 2370 mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq); 2371 2372 mqrq->brq.mrq.done = mmc_blk_mq_req_done; 2373 2374 mmc_pre_req(host, &mqrq->brq.mrq); 2375 2376 err = mmc_blk_rw_wait(mq, &prev_req); 2377 if (err) 2378 goto out_post_req; 2379 2380 mq->rw_wait = true; 2381 2382 err = mmc_start_request(host, &mqrq->brq.mrq); 2383 2384 if (prev_req) 2385 mmc_blk_mq_post_req(mq, prev_req, true); 2386 2387 if (err) 2388 mq->rw_wait = false; 2389 2390 /* Release re-tuning here where there is no synchronization required */ 2391 if (err || mmc_host_can_done_complete(host)) 2392 mmc_retune_release(host); 2393 2394out_post_req: 2395 if (err) 2396 mmc_post_req(host, &mqrq->brq.mrq, err); 2397 2398 return err; 2399} 2400 2401static int mmc_blk_wait_for_idle(struct mmc_queue *mq, struct mmc_host *host) 2402{ 2403 if (host->cqe_enabled) 2404 return host->cqe_ops->cqe_wait_for_idle(host); 2405 2406 return mmc_blk_rw_wait(mq, NULL); 2407} 2408 2409enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req) 2410{ 2411 struct mmc_blk_data *md = mq->blkdata; 2412 struct mmc_card *card = md->queue.card; 2413 struct mmc_host *host = card->host; 2414 int ret; 2415 2416 ret = mmc_blk_part_switch(card, md->part_type); 2417 if (ret) 2418 return MMC_REQ_FAILED_TO_START; 2419 2420 switch (mmc_issue_type(mq, req)) { 2421 case MMC_ISSUE_SYNC: 2422 ret = mmc_blk_wait_for_idle(mq, host); 2423 if (ret) 2424 return MMC_REQ_BUSY; 2425 switch (req_op(req)) { 2426 case REQ_OP_DRV_IN: 2427 case REQ_OP_DRV_OUT: 2428 mmc_blk_issue_drv_op(mq, req); 2429 break; 2430 case REQ_OP_DISCARD: 2431 mmc_blk_issue_discard_rq(mq, req); 2432 break; 2433 case REQ_OP_SECURE_ERASE: 2434 mmc_blk_issue_secdiscard_rq(mq, req); 2435 break; 2436 case REQ_OP_WRITE_ZEROES: 2437 mmc_blk_issue_trim_rq(mq, req); 2438 break; 2439 case REQ_OP_FLUSH: 2440 mmc_blk_issue_flush(mq, req); 2441 break; 2442 default: 2443 WARN_ON_ONCE(1); 2444 return MMC_REQ_FAILED_TO_START; 2445 } 2446 return MMC_REQ_FINISHED; 2447 case MMC_ISSUE_DCMD: 2448 case MMC_ISSUE_ASYNC: 2449 switch (req_op(req)) { 2450 case REQ_OP_FLUSH: 2451 if (!mmc_cache_enabled(host)) { 2452 blk_mq_end_request(req, BLK_STS_OK); 2453 return MMC_REQ_FINISHED; 2454 } 2455 ret = mmc_blk_cqe_issue_flush(mq, req); 2456 break; 2457 case REQ_OP_WRITE: 2458 card->written_flag = true; 2459 fallthrough; 2460 case REQ_OP_READ: 2461 if (host->cqe_enabled) 2462 ret = mmc_blk_cqe_issue_rw_rq(mq, req); 2463 else 2464 ret = mmc_blk_mq_issue_rw_rq(mq, req); 2465 break; 2466 default: 2467 WARN_ON_ONCE(1); 2468 ret = -EINVAL; 2469 } 2470 if (!ret) 2471 return MMC_REQ_STARTED; 2472 return ret == -EBUSY ? MMC_REQ_BUSY : MMC_REQ_FAILED_TO_START; 2473 default: 2474 WARN_ON_ONCE(1); 2475 return MMC_REQ_FAILED_TO_START; 2476 } 2477} 2478 2479static inline int mmc_blk_readonly(struct mmc_card *card) 2480{ 2481 return mmc_card_readonly(card) || 2482 !(card->csd.cmdclass & CCC_BLOCK_WRITE); 2483} 2484 2485/* 2486 * Search for a declared partitions node for the disk in mmc-card related node. 2487 * 2488 * This is to permit support for partition table defined in DT in special case 2489 * where a partition table is not written in the disk and is expected to be 2490 * passed from the running system. 2491 * 2492 * For the user disk, "partitions" node is searched. 2493 * For the special HW disk, "partitions-" node with the appended name is used 2494 * following this conversion table (to adhere to JEDEC naming) 2495 * - boot0 -> partitions-boot1 2496 * - boot1 -> partitions-boot2 2497 * - gp0 -> partitions-gp1 2498 * - gp1 -> partitions-gp2 2499 * - gp2 -> partitions-gp3 2500 * - gp3 -> partitions-gp4 2501 */ 2502static struct fwnode_handle *mmc_blk_get_partitions_node(struct device *mmc_dev, 2503 const char *subname) 2504{ 2505 const char *node_name = "partitions"; 2506 2507 if (subname) { 2508 mmc_dev = mmc_dev->parent; 2509 2510 /* 2511 * Check if we are allocating a BOOT disk boot0/1 disk. 2512 * In DT we use the JEDEC naming boot1/2. 2513 */ 2514 if (!strcmp(subname, "boot0")) 2515 node_name = "partitions-boot1"; 2516 if (!strcmp(subname, "boot1")) 2517 node_name = "partitions-boot2"; 2518 /* 2519 * Check if we are allocating a GP disk gp0/1/2/3 disk. 2520 * In DT we use the JEDEC naming gp1/2/3/4. 2521 */ 2522 if (!strcmp(subname, "gp0")) 2523 node_name = "partitions-gp1"; 2524 if (!strcmp(subname, "gp1")) 2525 node_name = "partitions-gp2"; 2526 if (!strcmp(subname, "gp2")) 2527 node_name = "partitions-gp3"; 2528 if (!strcmp(subname, "gp3")) 2529 node_name = "partitions-gp4"; 2530 } 2531 2532 return device_get_named_child_node(mmc_dev, node_name); 2533} 2534 2535static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card, 2536 struct device *parent, 2537 sector_t size, 2538 bool default_ro, 2539 const char *subname, 2540 int area_type, 2541 unsigned int part_type) 2542{ 2543 struct fwnode_handle *disk_fwnode; 2544 struct mmc_blk_data *md; 2545 int devidx, ret; 2546 char cap_str[10]; 2547 unsigned int features = 0; 2548 2549 devidx = ida_alloc_max(&mmc_blk_ida, max_devices - 1, GFP_KERNEL); 2550 if (devidx < 0) { 2551 /* 2552 * We get -ENOSPC because there are no more any available 2553 * devidx. The reason may be that, either userspace haven't yet 2554 * unmounted the partitions, which postpones mmc_blk_release() 2555 * from being called, or the device has more partitions than 2556 * what we support. 2557 */ 2558 if (devidx == -ENOSPC) 2559 dev_err(mmc_dev(card->host), 2560 "no more device IDs available\n"); 2561 2562 return ERR_PTR(devidx); 2563 } 2564 2565 md = kzalloc(sizeof(*md), GFP_KERNEL); 2566 if (!md) { 2567 ret = -ENOMEM; 2568 goto out; 2569 } 2570 2571 md->area_type = area_type; 2572 2573 /* 2574 * Set the read-only status based on the supported commands 2575 * and the write protect switch. 2576 */ 2577 md->read_only = mmc_blk_readonly(card); 2578 2579 if (mmc_host_can_cmd23(card->host) && mmc_card_can_cmd23(card)) 2580 md->flags |= MMC_BLK_CMD23; 2581 2582 if (md->flags & MMC_BLK_CMD23 && 2583 ((card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN) || 2584 card->ext_csd.rel_sectors)) { 2585 md->flags |= MMC_BLK_REL_WR; 2586 features |= (BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA); 2587 } else if (mmc_cache_enabled(card->host)) { 2588 features |= BLK_FEAT_WRITE_CACHE; 2589 } 2590 2591 md->disk = mmc_init_queue(&md->queue, card, features); 2592 if (IS_ERR(md->disk)) { 2593 ret = PTR_ERR(md->disk); 2594 goto err_kfree; 2595 } 2596 2597 INIT_LIST_HEAD(&md->part); 2598 INIT_LIST_HEAD(&md->rpmbs); 2599 kref_init(&md->kref); 2600 2601 md->queue.blkdata = md; 2602 md->part_type = part_type; 2603 2604 md->disk->major = MMC_BLOCK_MAJOR; 2605 md->disk->minors = perdev_minors; 2606 md->disk->first_minor = devidx * perdev_minors; 2607 md->disk->fops = &mmc_bdops; 2608 md->disk->private_data = md; 2609 md->parent = parent; 2610 set_disk_ro(md->disk, md->read_only || default_ro); 2611 if (area_type & MMC_BLK_DATA_AREA_RPMB) 2612 md->disk->flags |= GENHD_FL_NO_PART; 2613 2614 /* 2615 * As discussed on lkml, GENHD_FL_REMOVABLE should: 2616 * 2617 * - be set for removable media with permanent block devices 2618 * - be unset for removable block devices with permanent media 2619 * 2620 * Since MMC block devices clearly fall under the second 2621 * case, we do not set GENHD_FL_REMOVABLE. Userspace 2622 * should use the block device creation/destruction hotplug 2623 * messages to tell when the card is present. 2624 */ 2625 2626 snprintf(md->disk->disk_name, sizeof(md->disk->disk_name), 2627 "mmcblk%u%s", card->host->index, subname ? subname : ""); 2628 2629 set_capacity(md->disk, size); 2630 2631 string_get_size((u64)size, 512, STRING_UNITS_2, 2632 cap_str, sizeof(cap_str)); 2633 pr_info("%s: %s %s %s%s\n", 2634 md->disk->disk_name, mmc_card_id(card), mmc_card_name(card), 2635 cap_str, md->read_only ? " (ro)" : ""); 2636 2637 /* used in ->open, must be set before add_disk: */ 2638 if (area_type == MMC_BLK_DATA_AREA_MAIN) 2639 dev_set_drvdata(&card->dev, md); 2640 disk_fwnode = mmc_blk_get_partitions_node(parent, subname); 2641 ret = add_disk_fwnode(md->parent, md->disk, mmc_disk_attr_groups, 2642 disk_fwnode); 2643 if (ret) 2644 goto err_put_disk; 2645 return md; 2646 2647 err_put_disk: 2648 put_disk(md->disk); 2649 blk_mq_free_tag_set(&md->queue.tag_set); 2650 err_kfree: 2651 kfree(md); 2652 out: 2653 ida_free(&mmc_blk_ida, devidx); 2654 return ERR_PTR(ret); 2655} 2656 2657static struct mmc_blk_data *mmc_blk_alloc(struct mmc_card *card) 2658{ 2659 sector_t size; 2660 2661 if (!mmc_card_sd(card) && mmc_card_blockaddr(card)) { 2662 /* 2663 * The EXT_CSD sector count is in number or 512 byte 2664 * sectors. 2665 */ 2666 size = card->ext_csd.sectors; 2667 } else { 2668 /* 2669 * The CSD capacity field is in units of read_blkbits. 2670 * set_capacity takes units of 512 bytes. 2671 */ 2672 size = (typeof(sector_t))card->csd.capacity 2673 << (card->csd.read_blkbits - 9); 2674 } 2675 2676 return mmc_blk_alloc_req(card, &card->dev, size, false, NULL, 2677 MMC_BLK_DATA_AREA_MAIN, 0); 2678} 2679 2680static int mmc_blk_alloc_part(struct mmc_card *card, 2681 struct mmc_blk_data *md, 2682 unsigned int part_type, 2683 sector_t size, 2684 bool default_ro, 2685 const char *subname, 2686 int area_type) 2687{ 2688 struct mmc_blk_data *part_md; 2689 2690 part_md = mmc_blk_alloc_req(card, disk_to_dev(md->disk), size, default_ro, 2691 subname, area_type, part_type); 2692 if (IS_ERR(part_md)) 2693 return PTR_ERR(part_md); 2694 list_add(&part_md->part, &md->part); 2695 2696 return 0; 2697} 2698 2699/** 2700 * mmc_rpmb_ioctl() - ioctl handler for the RPMB chardev 2701 * @filp: the character device file 2702 * @cmd: the ioctl() command 2703 * @arg: the argument from userspace 2704 * 2705 * This will essentially just redirect the ioctl()s coming in over to 2706 * the main block device spawning the RPMB character device. 2707 */ 2708static long mmc_rpmb_ioctl(struct file *filp, unsigned int cmd, 2709 unsigned long arg) 2710{ 2711 struct mmc_rpmb_data *rpmb = filp->private_data; 2712 int ret; 2713 2714 switch (cmd) { 2715 case MMC_IOC_CMD: 2716 ret = mmc_blk_ioctl_cmd(rpmb->md, 2717 (struct mmc_ioc_cmd __user *)arg, 2718 rpmb); 2719 break; 2720 case MMC_IOC_MULTI_CMD: 2721 ret = mmc_blk_ioctl_multi_cmd(rpmb->md, 2722 (struct mmc_ioc_multi_cmd __user *)arg, 2723 rpmb); 2724 break; 2725 default: 2726 ret = -EINVAL; 2727 break; 2728 } 2729 2730 return ret; 2731} 2732 2733#ifdef CONFIG_COMPAT 2734static long mmc_rpmb_ioctl_compat(struct file *filp, unsigned int cmd, 2735 unsigned long arg) 2736{ 2737 return mmc_rpmb_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)); 2738} 2739#endif 2740 2741static int mmc_rpmb_chrdev_open(struct inode *inode, struct file *filp) 2742{ 2743 struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev, 2744 struct mmc_rpmb_data, chrdev); 2745 2746 get_device(&rpmb->dev); 2747 filp->private_data = rpmb; 2748 2749 return nonseekable_open(inode, filp); 2750} 2751 2752static int mmc_rpmb_chrdev_release(struct inode *inode, struct file *filp) 2753{ 2754 struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev, 2755 struct mmc_rpmb_data, chrdev); 2756 2757 put_device(&rpmb->dev); 2758 2759 return 0; 2760} 2761 2762static const struct file_operations mmc_rpmb_fileops = { 2763 .release = mmc_rpmb_chrdev_release, 2764 .open = mmc_rpmb_chrdev_open, 2765 .owner = THIS_MODULE, 2766 .unlocked_ioctl = mmc_rpmb_ioctl, 2767#ifdef CONFIG_COMPAT 2768 .compat_ioctl = mmc_rpmb_ioctl_compat, 2769#endif 2770}; 2771 2772static void mmc_blk_rpmb_device_release(struct device *dev) 2773{ 2774 struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev); 2775 2776 rpmb_dev_unregister(rpmb->rdev); 2777 mmc_blk_put(rpmb->md); 2778 ida_free(&mmc_rpmb_ida, rpmb->id); 2779 kfree(rpmb); 2780} 2781 2782static void free_idata(struct mmc_blk_ioc_data **idata, unsigned int cmd_count) 2783{ 2784 unsigned int n; 2785 2786 for (n = 0; n < cmd_count; n++) 2787 kfree(idata[n]); 2788 kfree(idata); 2789} 2790 2791static struct mmc_blk_ioc_data **alloc_idata(struct mmc_rpmb_data *rpmb, 2792 unsigned int cmd_count) 2793{ 2794 struct mmc_blk_ioc_data **idata; 2795 unsigned int n; 2796 2797 idata = kcalloc(cmd_count, sizeof(*idata), GFP_KERNEL); 2798 if (!idata) 2799 return NULL; 2800 2801 for (n = 0; n < cmd_count; n++) { 2802 idata[n] = kcalloc(1, sizeof(**idata), GFP_KERNEL); 2803 if (!idata[n]) { 2804 free_idata(idata, n); 2805 return NULL; 2806 } 2807 idata[n]->rpmb = rpmb; 2808 } 2809 2810 return idata; 2811} 2812 2813static void set_idata(struct mmc_blk_ioc_data *idata, u32 opcode, 2814 int write_flag, u8 *buf, unsigned int buf_bytes) 2815{ 2816 /* 2817 * The size of an RPMB frame must match what's expected by the 2818 * hardware. 2819 */ 2820 static_assert(!CHECK_SIZE_NEQ(512), "RPMB frame size must be 512 bytes"); 2821 2822 idata->ic.opcode = opcode; 2823 idata->ic.flags = MMC_RSP_R1 | MMC_CMD_ADTC; 2824 idata->ic.write_flag = write_flag; 2825 idata->ic.blksz = RPMB_FRAME_SIZE; 2826 idata->ic.blocks = buf_bytes / idata->ic.blksz; 2827 idata->buf = buf; 2828 idata->buf_bytes = buf_bytes; 2829} 2830 2831static int mmc_route_rpmb_frames(struct device *dev, u8 *req, 2832 unsigned int req_len, u8 *resp, 2833 unsigned int resp_len) 2834{ 2835 struct rpmb_frame *frm = (struct rpmb_frame *)req; 2836 struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev); 2837 struct mmc_blk_data *md = rpmb->md; 2838 struct mmc_blk_ioc_data **idata; 2839 struct mmc_queue_req *mq_rq; 2840 unsigned int cmd_count; 2841 struct request *rq; 2842 u16 req_type; 2843 bool write; 2844 int ret; 2845 2846 if (IS_ERR(md->queue.card)) 2847 return PTR_ERR(md->queue.card); 2848 2849 if (req_len < RPMB_FRAME_SIZE) 2850 return -EINVAL; 2851 2852 req_type = be16_to_cpu(frm->req_resp); 2853 switch (req_type) { 2854 case RPMB_PROGRAM_KEY: 2855 if (CHECK_SIZE_NEQ(req_len) || CHECK_SIZE_NEQ(resp_len)) 2856 return -EINVAL; 2857 write = true; 2858 break; 2859 case RPMB_GET_WRITE_COUNTER: 2860 if (CHECK_SIZE_NEQ(req_len) || CHECK_SIZE_NEQ(resp_len)) 2861 return -EINVAL; 2862 write = false; 2863 break; 2864 case RPMB_WRITE_DATA: 2865 if (!CHECK_SIZE_ALIGNED(req_len) || CHECK_SIZE_NEQ(resp_len)) 2866 return -EINVAL; 2867 write = true; 2868 break; 2869 case RPMB_READ_DATA: 2870 if (CHECK_SIZE_NEQ(req_len) || !CHECK_SIZE_ALIGNED(resp_len)) 2871 return -EINVAL; 2872 write = false; 2873 break; 2874 default: 2875 return -EINVAL; 2876 } 2877 2878 /* Write operations require 3 commands, read operations require 2 */ 2879 cmd_count = write ? 3 : 2; 2880 2881 idata = alloc_idata(rpmb, cmd_count); 2882 if (!idata) 2883 return -ENOMEM; 2884 2885 if (write) { 2886 struct rpmb_frame *resp_frm = (struct rpmb_frame *)resp; 2887 2888 /* Send write request frame(s) */ 2889 set_idata(idata[0], MMC_WRITE_MULTIPLE_BLOCK, 2890 1 | MMC_CMD23_ARG_REL_WR, req, req_len); 2891 2892 /* Send result request frame */ 2893 memset(resp_frm, 0, RPMB_FRAME_SIZE); 2894 resp_frm->req_resp = cpu_to_be16(RPMB_RESULT_READ); 2895 set_idata(idata[1], MMC_WRITE_MULTIPLE_BLOCK, 1, resp, 2896 resp_len); 2897 2898 /* Read response frame */ 2899 set_idata(idata[2], MMC_READ_MULTIPLE_BLOCK, 0, resp, resp_len); 2900 } else { 2901 /* Send write request frame(s) */ 2902 set_idata(idata[0], MMC_WRITE_MULTIPLE_BLOCK, 1, req, req_len); 2903 2904 /* Read response frame */ 2905 set_idata(idata[1], MMC_READ_MULTIPLE_BLOCK, 0, resp, resp_len); 2906 } 2907 2908 rq = blk_mq_alloc_request(md->queue.queue, REQ_OP_DRV_OUT, 0); 2909 if (IS_ERR(rq)) { 2910 ret = PTR_ERR(rq); 2911 goto out; 2912 } 2913 2914 mq_rq = req_to_mmc_queue_req(rq); 2915 mq_rq->drv_op = MMC_DRV_OP_IOCTL_RPMB; 2916 mq_rq->drv_op_result = -EIO; 2917 mq_rq->drv_op_data = idata; 2918 mq_rq->ioc_count = cmd_count; 2919 blk_execute_rq(rq, false); 2920 ret = req_to_mmc_queue_req(rq)->drv_op_result; 2921 2922 blk_mq_free_request(rq); 2923 2924out: 2925 free_idata(idata, cmd_count); 2926 return ret; 2927} 2928 2929static int mmc_blk_alloc_rpmb_part(struct mmc_card *card, 2930 struct mmc_blk_data *md, 2931 unsigned int part_index, 2932 sector_t size, 2933 const char *subname) 2934{ 2935 int devidx, ret; 2936 char rpmb_name[DISK_NAME_LEN]; 2937 char cap_str[10]; 2938 struct mmc_rpmb_data *rpmb; 2939 2940 /* This creates the minor number for the RPMB char device */ 2941 devidx = ida_alloc_max(&mmc_rpmb_ida, max_devices - 1, GFP_KERNEL); 2942 if (devidx < 0) 2943 return devidx; 2944 2945 rpmb = kzalloc(sizeof(*rpmb), GFP_KERNEL); 2946 if (!rpmb) { 2947 ida_free(&mmc_rpmb_ida, devidx); 2948 return -ENOMEM; 2949 } 2950 2951 snprintf(rpmb_name, sizeof(rpmb_name), 2952 "mmcblk%u%s", card->host->index, subname ? subname : ""); 2953 2954 rpmb->id = devidx; 2955 rpmb->part_index = part_index; 2956 rpmb->dev.init_name = rpmb_name; 2957 rpmb->dev.bus = &mmc_rpmb_bus_type; 2958 rpmb->dev.devt = MKDEV(MAJOR(mmc_rpmb_devt), rpmb->id); 2959 rpmb->dev.parent = &card->dev; 2960 rpmb->dev.release = mmc_blk_rpmb_device_release; 2961 device_initialize(&rpmb->dev); 2962 dev_set_drvdata(&rpmb->dev, rpmb); 2963 mmc_blk_get(md->disk); 2964 rpmb->md = md; 2965 2966 cdev_init(&rpmb->chrdev, &mmc_rpmb_fileops); 2967 rpmb->chrdev.owner = THIS_MODULE; 2968 ret = cdev_device_add(&rpmb->chrdev, &rpmb->dev); 2969 if (ret) { 2970 pr_err("%s: could not add character device\n", rpmb_name); 2971 goto out_put_device; 2972 } 2973 2974 list_add(&rpmb->node, &md->rpmbs); 2975 2976 string_get_size((u64)size, 512, STRING_UNITS_2, 2977 cap_str, sizeof(cap_str)); 2978 2979 pr_info("%s: %s %s %s, chardev (%d:%d)\n", 2980 rpmb_name, mmc_card_id(card), mmc_card_name(card), cap_str, 2981 MAJOR(mmc_rpmb_devt), rpmb->id); 2982 2983 return 0; 2984 2985out_put_device: 2986 put_device(&rpmb->dev); 2987 return ret; 2988} 2989 2990static void mmc_blk_remove_rpmb_part(struct mmc_rpmb_data *rpmb) 2991 2992{ 2993 cdev_device_del(&rpmb->chrdev, &rpmb->dev); 2994 put_device(&rpmb->dev); 2995} 2996 2997/* MMC Physical partitions consist of two boot partitions and 2998 * up to four general purpose partitions. 2999 * For each partition enabled in EXT_CSD a block device will be allocatedi 3000 * to provide access to the partition. 3001 */ 3002 3003static int mmc_blk_alloc_parts(struct mmc_card *card, struct mmc_blk_data *md) 3004{ 3005 int idx, ret; 3006 3007 if (!mmc_card_mmc(card)) 3008 return 0; 3009 3010 for (idx = 0; idx < card->nr_parts; idx++) { 3011 if (card->part[idx].area_type & MMC_BLK_DATA_AREA_RPMB) { 3012 /* 3013 * RPMB partitions does not provide block access, they 3014 * are only accessed using ioctl():s. Thus create 3015 * special RPMB block devices that do not have a 3016 * backing block queue for these. 3017 */ 3018 ret = mmc_blk_alloc_rpmb_part(card, md, 3019 card->part[idx].part_cfg, 3020 card->part[idx].size >> 9, 3021 card->part[idx].name); 3022 if (ret) 3023 return ret; 3024 } else if (card->part[idx].size) { 3025 ret = mmc_blk_alloc_part(card, md, 3026 card->part[idx].part_cfg, 3027 card->part[idx].size >> 9, 3028 card->part[idx].force_ro, 3029 card->part[idx].name, 3030 card->part[idx].area_type); 3031 if (ret) 3032 return ret; 3033 } 3034 } 3035 3036 return 0; 3037} 3038 3039static void mmc_blk_remove_req(struct mmc_blk_data *md) 3040{ 3041 /* 3042 * Flush remaining requests and free queues. It is freeing the queue 3043 * that stops new requests from being accepted. 3044 */ 3045 del_gendisk(md->disk); 3046 mmc_cleanup_queue(&md->queue); 3047 mmc_blk_put(md); 3048} 3049 3050static void mmc_blk_remove_parts(struct mmc_card *card, 3051 struct mmc_blk_data *md) 3052{ 3053 struct list_head *pos, *q; 3054 struct mmc_blk_data *part_md; 3055 struct mmc_rpmb_data *rpmb; 3056 3057 /* Remove RPMB partitions */ 3058 list_for_each_safe(pos, q, &md->rpmbs) { 3059 rpmb = list_entry(pos, struct mmc_rpmb_data, node); 3060 list_del(pos); 3061 mmc_blk_remove_rpmb_part(rpmb); 3062 } 3063 /* Remove block partitions */ 3064 list_for_each_safe(pos, q, &md->part) { 3065 part_md = list_entry(pos, struct mmc_blk_data, part); 3066 list_del(pos); 3067 mmc_blk_remove_req(part_md); 3068 } 3069} 3070 3071#ifdef CONFIG_DEBUG_FS 3072 3073static int mmc_dbg_card_status_get(void *data, u64 *val) 3074{ 3075 struct mmc_card *card = data; 3076 struct mmc_blk_data *md = dev_get_drvdata(&card->dev); 3077 struct mmc_queue *mq = &md->queue; 3078 struct request *req; 3079 int ret; 3080 3081 /* Ask the block layer about the card status */ 3082 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0); 3083 if (IS_ERR(req)) 3084 return PTR_ERR(req); 3085 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS; 3086 req_to_mmc_queue_req(req)->drv_op_result = -EIO; 3087 blk_execute_rq(req, false); 3088 ret = req_to_mmc_queue_req(req)->drv_op_result; 3089 if (ret >= 0) { 3090 *val = ret; 3091 ret = 0; 3092 } 3093 blk_mq_free_request(req); 3094 3095 return ret; 3096} 3097DEFINE_DEBUGFS_ATTRIBUTE(mmc_dbg_card_status_fops, mmc_dbg_card_status_get, 3098 NULL, "%08llx\n"); 3099 3100/* That is two digits * 512 + 1 for newline */ 3101#define EXT_CSD_STR_LEN 1025 3102 3103static int mmc_ext_csd_open(struct inode *inode, struct file *filp) 3104{ 3105 struct mmc_card *card = inode->i_private; 3106 struct mmc_blk_data *md = dev_get_drvdata(&card->dev); 3107 struct mmc_queue *mq = &md->queue; 3108 struct request *req; 3109 char *buf; 3110 ssize_t n = 0; 3111 u8 *ext_csd; 3112 int err, i; 3113 3114 buf = kmalloc(EXT_CSD_STR_LEN + 1, GFP_KERNEL); 3115 if (!buf) 3116 return -ENOMEM; 3117 3118 /* Ask the block layer for the EXT CSD */ 3119 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0); 3120 if (IS_ERR(req)) { 3121 err = PTR_ERR(req); 3122 goto out_free; 3123 } 3124 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD; 3125 req_to_mmc_queue_req(req)->drv_op_result = -EIO; 3126 req_to_mmc_queue_req(req)->drv_op_data = &ext_csd; 3127 blk_execute_rq(req, false); 3128 err = req_to_mmc_queue_req(req)->drv_op_result; 3129 blk_mq_free_request(req); 3130 if (err) { 3131 pr_err("FAILED %d\n", err); 3132 goto out_free; 3133 } 3134 3135 for (i = 0; i < 512; i++) 3136 n += sprintf(buf + n, "%02x", ext_csd[i]); 3137 n += sprintf(buf + n, "\n"); 3138 3139 if (n != EXT_CSD_STR_LEN) { 3140 err = -EINVAL; 3141 kfree(ext_csd); 3142 goto out_free; 3143 } 3144 3145 filp->private_data = buf; 3146 kfree(ext_csd); 3147 return 0; 3148 3149out_free: 3150 kfree(buf); 3151 return err; 3152} 3153 3154static ssize_t mmc_ext_csd_read(struct file *filp, char __user *ubuf, 3155 size_t cnt, loff_t *ppos) 3156{ 3157 char *buf = filp->private_data; 3158 3159 return simple_read_from_buffer(ubuf, cnt, ppos, 3160 buf, EXT_CSD_STR_LEN); 3161} 3162 3163static int mmc_ext_csd_release(struct inode *inode, struct file *file) 3164{ 3165 kfree(file->private_data); 3166 return 0; 3167} 3168 3169static const struct file_operations mmc_dbg_ext_csd_fops = { 3170 .open = mmc_ext_csd_open, 3171 .read = mmc_ext_csd_read, 3172 .release = mmc_ext_csd_release, 3173 .llseek = default_llseek, 3174}; 3175 3176static void mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md) 3177{ 3178 struct dentry *root; 3179 3180 if (!card->debugfs_root) 3181 return; 3182 3183 root = card->debugfs_root; 3184 3185 if (mmc_card_mmc(card) || mmc_card_sd(card)) { 3186 md->status_dentry = 3187 debugfs_create_file_unsafe("status", 0400, root, 3188 card, 3189 &mmc_dbg_card_status_fops); 3190 } 3191 3192 if (mmc_card_mmc(card)) { 3193 md->ext_csd_dentry = 3194 debugfs_create_file("ext_csd", 0400, root, card, 3195 &mmc_dbg_ext_csd_fops); 3196 } 3197} 3198 3199static void mmc_blk_remove_debugfs(struct mmc_card *card, 3200 struct mmc_blk_data *md) 3201{ 3202 if (!card->debugfs_root) 3203 return; 3204 3205 debugfs_remove(md->status_dentry); 3206 md->status_dentry = NULL; 3207 3208 debugfs_remove(md->ext_csd_dentry); 3209 md->ext_csd_dentry = NULL; 3210} 3211 3212#else 3213 3214static void mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md) 3215{ 3216} 3217 3218static void mmc_blk_remove_debugfs(struct mmc_card *card, 3219 struct mmc_blk_data *md) 3220{ 3221} 3222 3223#endif /* CONFIG_DEBUG_FS */ 3224 3225static void mmc_blk_rpmb_add(struct mmc_card *card) 3226{ 3227 struct mmc_blk_data *md = dev_get_drvdata(&card->dev); 3228 struct mmc_rpmb_data *rpmb; 3229 struct rpmb_dev *rdev; 3230 unsigned int n; 3231 u32 cid[4]; 3232 struct rpmb_descr descr = { 3233 .type = RPMB_TYPE_EMMC, 3234 .route_frames = mmc_route_rpmb_frames, 3235 .reliable_wr_count = card->ext_csd.enhanced_rpmb_supported ? 3236 2 : 32, 3237 .capacity = card->ext_csd.raw_rpmb_size_mult, 3238 .dev_id = (void *)cid, 3239 .dev_id_len = sizeof(cid), 3240 }; 3241 3242 /* 3243 * Provice CID as an octet array. The CID needs to be interpreted 3244 * when used as input to derive the RPMB key since some fields 3245 * will change due to firmware updates. 3246 */ 3247 for (n = 0; n < 4; n++) 3248 cid[n] = be32_to_cpu((__force __be32)card->raw_cid[n]); 3249 3250 list_for_each_entry(rpmb, &md->rpmbs, node) { 3251 rdev = rpmb_dev_register(&rpmb->dev, &descr); 3252 if (IS_ERR(rdev)) { 3253 pr_warn("%s: could not register RPMB device\n", 3254 dev_name(&rpmb->dev)); 3255 continue; 3256 } 3257 rpmb->rdev = rdev; 3258 } 3259} 3260 3261static int mmc_blk_probe(struct mmc_card *card) 3262{ 3263 struct mmc_blk_data *md; 3264 int ret = 0; 3265 3266 /* 3267 * Check that the card supports the command class(es) we need. 3268 */ 3269 if (!(card->csd.cmdclass & CCC_BLOCK_READ)) 3270 return -ENODEV; 3271 3272 mmc_fixup_device(card, mmc_blk_fixups); 3273 3274 card->complete_wq = alloc_workqueue("mmc_complete", 3275 WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 3276 0); 3277 if (!card->complete_wq) { 3278 pr_err("Failed to create mmc completion workqueue"); 3279 return -ENOMEM; 3280 } 3281 3282 md = mmc_blk_alloc(card); 3283 if (IS_ERR(md)) { 3284 ret = PTR_ERR(md); 3285 goto out_free; 3286 } 3287 3288 ret = mmc_blk_alloc_parts(card, md); 3289 if (ret) 3290 goto out; 3291 3292 /* Add two debugfs entries */ 3293 mmc_blk_add_debugfs(card, md); 3294 3295 pm_runtime_set_autosuspend_delay(&card->dev, 3000); 3296 pm_runtime_use_autosuspend(&card->dev); 3297 3298 /* 3299 * Don't enable runtime PM for SD-combo cards here. Leave that 3300 * decision to be taken during the SDIO init sequence instead. 3301 */ 3302 if (!mmc_card_sd_combo(card)) { 3303 pm_runtime_set_active(&card->dev); 3304 pm_runtime_enable(&card->dev); 3305 } 3306 3307 mmc_blk_rpmb_add(card); 3308 3309 return 0; 3310 3311out: 3312 mmc_blk_remove_parts(card, md); 3313 mmc_blk_remove_req(md); 3314out_free: 3315 destroy_workqueue(card->complete_wq); 3316 return ret; 3317} 3318 3319static void mmc_blk_remove(struct mmc_card *card) 3320{ 3321 struct mmc_blk_data *md = dev_get_drvdata(&card->dev); 3322 3323 mmc_blk_remove_debugfs(card, md); 3324 mmc_blk_remove_parts(card, md); 3325 pm_runtime_get_sync(&card->dev); 3326 if (md->part_curr != md->part_type) { 3327 mmc_claim_host(card->host); 3328 mmc_blk_part_switch(card, md->part_type); 3329 mmc_release_host(card->host); 3330 } 3331 if (!mmc_card_sd_combo(card)) 3332 pm_runtime_disable(&card->dev); 3333 pm_runtime_put_noidle(&card->dev); 3334 mmc_blk_remove_req(md); 3335 destroy_workqueue(card->complete_wq); 3336} 3337 3338static int _mmc_blk_suspend(struct mmc_card *card) 3339{ 3340 struct mmc_blk_data *part_md; 3341 struct mmc_blk_data *md = dev_get_drvdata(&card->dev); 3342 3343 if (md) { 3344 mmc_queue_suspend(&md->queue); 3345 list_for_each_entry(part_md, &md->part, part) { 3346 mmc_queue_suspend(&part_md->queue); 3347 } 3348 } 3349 return 0; 3350} 3351 3352static void mmc_blk_shutdown(struct mmc_card *card) 3353{ 3354 _mmc_blk_suspend(card); 3355} 3356 3357#ifdef CONFIG_PM_SLEEP 3358static int mmc_blk_suspend(struct device *dev) 3359{ 3360 struct mmc_card *card = mmc_dev_to_card(dev); 3361 3362 return _mmc_blk_suspend(card); 3363} 3364 3365static int mmc_blk_resume(struct device *dev) 3366{ 3367 struct mmc_blk_data *part_md; 3368 struct mmc_blk_data *md = dev_get_drvdata(dev); 3369 3370 if (md) { 3371 /* 3372 * Resume involves the card going into idle state, 3373 * so current partition is always the main one. 3374 */ 3375 md->part_curr = md->part_type; 3376 mmc_queue_resume(&md->queue); 3377 list_for_each_entry(part_md, &md->part, part) { 3378 mmc_queue_resume(&part_md->queue); 3379 } 3380 } 3381 return 0; 3382} 3383#endif 3384 3385static SIMPLE_DEV_PM_OPS(mmc_blk_pm_ops, mmc_blk_suspend, mmc_blk_resume); 3386 3387static struct mmc_driver mmc_driver = { 3388 .drv = { 3389 .name = "mmcblk", 3390 .pm = &mmc_blk_pm_ops, 3391 }, 3392 .probe = mmc_blk_probe, 3393 .remove = mmc_blk_remove, 3394 .shutdown = mmc_blk_shutdown, 3395}; 3396 3397static int __init mmc_blk_init(void) 3398{ 3399 int res; 3400 3401 res = bus_register(&mmc_rpmb_bus_type); 3402 if (res < 0) { 3403 pr_err("mmcblk: could not register RPMB bus type\n"); 3404 return res; 3405 } 3406 res = alloc_chrdev_region(&mmc_rpmb_devt, 0, MAX_DEVICES, "rpmb"); 3407 if (res < 0) { 3408 pr_err("mmcblk: failed to allocate rpmb chrdev region\n"); 3409 goto out_bus_unreg; 3410 } 3411 3412 if (perdev_minors != CONFIG_MMC_BLOCK_MINORS) 3413 pr_info("mmcblk: using %d minors per device\n", perdev_minors); 3414 3415 max_devices = min(MAX_DEVICES, (1 << MINORBITS) / perdev_minors); 3416 3417 res = register_blkdev(MMC_BLOCK_MAJOR, "mmc"); 3418 if (res) 3419 goto out_chrdev_unreg; 3420 3421 res = mmc_register_driver(&mmc_driver); 3422 if (res) 3423 goto out_blkdev_unreg; 3424 3425 return 0; 3426 3427out_blkdev_unreg: 3428 unregister_blkdev(MMC_BLOCK_MAJOR, "mmc"); 3429out_chrdev_unreg: 3430 unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES); 3431out_bus_unreg: 3432 bus_unregister(&mmc_rpmb_bus_type); 3433 return res; 3434} 3435 3436static void __exit mmc_blk_exit(void) 3437{ 3438 mmc_unregister_driver(&mmc_driver); 3439 unregister_blkdev(MMC_BLOCK_MAJOR, "mmc"); 3440 unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES); 3441 bus_unregister(&mmc_rpmb_bus_type); 3442} 3443 3444module_init(mmc_blk_init); 3445module_exit(mmc_blk_exit); 3446 3447MODULE_LICENSE("GPL"); 3448MODULE_DESCRIPTION("Multimedia Card (MMC) block device driver");
~/
Read-only file preview.