FILE
ufshcd.c
File preview with syntax highlighting.
linux-6.19.14/drivers/ufs/core/ufshcd.c
1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Universal Flash Storage Host controller driver Core 4 * Copyright (C) 2011-2013 Samsung India Software Operations 5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. 6 * 7 * Authors: 8 * Santosh Yaraganavi <santosh.sy@samsung.com> 9 * Vinayak Holikatti <h.vinayak@samsung.com> 10 */ 11 12#include <linux/async.h> 13#include <linux/devfreq.h> 14#include <linux/nls.h> 15#include <linux/of.h> 16#include <linux/bitfield.h> 17#include <linux/blk-pm.h> 18#include <linux/blkdev.h> 19#include <linux/clk.h> 20#include <linux/delay.h> 21#include <linux/interrupt.h> 22#include <linux/module.h> 23#include <linux/pm_opp.h> 24#include <linux/regulator/consumer.h> 25#include <linux/sched/clock.h> 26#include <linux/sizes.h> 27#include <linux/iopoll.h> 28#include <scsi/scsi_cmnd.h> 29#include <scsi/scsi_dbg.h> 30#include <scsi/scsi_driver.h> 31#include <scsi/scsi_eh.h> 32#include <scsi/scsi_tcq.h> 33#include "ufshcd-priv.h" 34#include <ufs/ufs_quirks.h> 35#include <ufs/unipro.h> 36#include "ufs-sysfs.h" 37#include "ufs-debugfs.h" 38#include "ufs-fault-injection.h" 39#include "ufs_bsg.h" 40#include "ufshcd-crypto.h" 41#include <linux/unaligned.h> 42 43#define CREATE_TRACE_POINTS 44#include "ufs_trace.h" 45 46#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ 47 UTP_TASK_REQ_COMPL |\ 48 UFSHCD_ERROR_MASK) 49 50/* UIC command timeout, unit: ms */ 51enum { 52 UIC_CMD_TIMEOUT_DEFAULT = 500, 53 UIC_CMD_TIMEOUT_MAX = 5000, 54}; 55/* NOP OUT retries waiting for NOP IN response */ 56#define NOP_OUT_RETRIES 10 57/* Timeout after 50 msecs if NOP OUT hangs without response */ 58#define NOP_OUT_TIMEOUT 50 /* msecs */ 59 60/* Query request retries */ 61#define QUERY_REQ_RETRIES 3 62/* Query request timeout */ 63enum { 64 QUERY_REQ_TIMEOUT_MIN = 1, 65 QUERY_REQ_TIMEOUT_DEFAULT = 1500, 66 QUERY_REQ_TIMEOUT_MAX = 30000 67}; 68 69/* Advanced RPMB request timeout */ 70#define ADVANCED_RPMB_REQ_TIMEOUT 3000 /* 3 seconds */ 71 72/* Task management command timeout */ 73#define TM_CMD_TIMEOUT 100 /* msecs */ 74 75/* maximum number of retries for a general UIC command */ 76#define UFS_UIC_COMMAND_RETRIES 3 77 78/* maximum number of link-startup retries */ 79#define DME_LINKSTARTUP_RETRIES 3 80 81/* maximum number of reset retries before giving up */ 82#define MAX_HOST_RESET_RETRIES 5 83 84/* Maximum number of error handler retries before giving up */ 85#define MAX_ERR_HANDLER_RETRIES 5 86 87/* Expose the flag value from utp_upiu_query.value */ 88#define MASK_QUERY_UPIU_FLAG_LOC 0xFF 89 90/* Interrupt aggregation default timeout, unit: 40us */ 91#define INT_AGGR_DEF_TO 0x02 92 93/* default delay of autosuspend: 2000 ms */ 94#define RPM_AUTOSUSPEND_DELAY_MS 2000 95 96/* Default delay of RPM device flush delayed work */ 97#define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000 98 99/* Default value of wait time before gating device ref clock */ 100#define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */ 101 102/* Polling time to wait for fDeviceInit */ 103#define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */ 104 105/* Default RTC update every 10 seconds */ 106#define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC) 107 108/* bMaxNumOfRTT is equal to two after device manufacturing */ 109#define DEFAULT_MAX_NUM_RTT 2 110 111/* UFSHC 4.0 compliant HC support this mode. */ 112static bool use_mcq_mode = true; 113 114static bool is_mcq_supported(struct ufs_hba *hba) 115{ 116 return hba->mcq_sup && use_mcq_mode; 117} 118 119module_param(use_mcq_mode, bool, 0644); 120MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default"); 121 122static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_DEFAULT; 123 124static int uic_cmd_timeout_set(const char *val, const struct kernel_param *kp) 125{ 126 return param_set_uint_minmax(val, kp, UIC_CMD_TIMEOUT_DEFAULT, 127 UIC_CMD_TIMEOUT_MAX); 128} 129 130static const struct kernel_param_ops uic_cmd_timeout_ops = { 131 .set = uic_cmd_timeout_set, 132 .get = param_get_uint, 133}; 134 135module_param_cb(uic_cmd_timeout, &uic_cmd_timeout_ops, &uic_cmd_timeout, 0644); 136MODULE_PARM_DESC(uic_cmd_timeout, 137 "UFS UIC command timeout in milliseconds. Defaults to 500ms. Supported values range from 500ms to 5 seconds inclusively"); 138 139static unsigned int dev_cmd_timeout = QUERY_REQ_TIMEOUT_DEFAULT; 140 141static int dev_cmd_timeout_set(const char *val, const struct kernel_param *kp) 142{ 143 return param_set_uint_minmax(val, kp, QUERY_REQ_TIMEOUT_MIN, 144 QUERY_REQ_TIMEOUT_MAX); 145} 146 147static const struct kernel_param_ops dev_cmd_timeout_ops = { 148 .set = dev_cmd_timeout_set, 149 .get = param_get_uint, 150}; 151 152module_param_cb(dev_cmd_timeout, &dev_cmd_timeout_ops, &dev_cmd_timeout, 0644); 153MODULE_PARM_DESC(dev_cmd_timeout, 154 "UFS Device command timeout in milliseconds. Defaults to 1.5s. Supported values range from 1ms to 30 seconds inclusively"); 155 156#define ufshcd_toggle_vreg(_dev, _vreg, _on) \ 157 ({ \ 158 int _ret; \ 159 if (_on) \ 160 _ret = ufshcd_enable_vreg(_dev, _vreg); \ 161 else \ 162 _ret = ufshcd_disable_vreg(_dev, _vreg); \ 163 _ret; \ 164 }) 165 166#define ufshcd_hex_dump(prefix_str, buf, len) do { \ 167 size_t __len = (len); \ 168 print_hex_dump(KERN_ERR, prefix_str, \ 169 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\ 170 16, 4, buf, __len, false); \ 171} while (0) 172 173int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len, 174 const char *prefix) 175{ 176 u32 *regs; 177 size_t pos; 178 179 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */ 180 return -EINVAL; 181 182 regs = kzalloc(len, GFP_ATOMIC); 183 if (!regs) 184 return -ENOMEM; 185 186 for (pos = 0; pos < len; pos += 4) { 187 if (offset == 0 && 188 pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER && 189 pos <= REG_UIC_ERROR_CODE_DME) 190 continue; 191 regs[pos / 4] = ufshcd_readl(hba, offset + pos); 192 } 193 194 ufshcd_hex_dump(prefix, regs, len); 195 kfree(regs); 196 197 return 0; 198} 199EXPORT_SYMBOL_GPL(ufshcd_dump_regs); 200 201enum { 202 UFSHCD_MAX_CHANNEL = 0, 203 UFSHCD_MAX_ID = 1, 204}; 205 206static const char *const ufshcd_state_name[] = { 207 [UFSHCD_STATE_RESET] = "reset", 208 [UFSHCD_STATE_OPERATIONAL] = "operational", 209 [UFSHCD_STATE_ERROR] = "error", 210 [UFSHCD_STATE_EH_SCHEDULED_FATAL] = "eh_fatal", 211 [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL] = "eh_non_fatal", 212}; 213 214/* UFSHCD error handling flags */ 215enum { 216 UFSHCD_EH_IN_PROGRESS = (1 << 0), 217}; 218 219/* UFSHCD UIC layer error flags */ 220enum { 221 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */ 222 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */ 223 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */ 224 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */ 225 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */ 226 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */ 227 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */ 228}; 229 230#define ufshcd_set_eh_in_progress(h) \ 231 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS) 232#define ufshcd_eh_in_progress(h) \ 233 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS) 234#define ufshcd_clear_eh_in_progress(h) \ 235 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS) 236 237const struct ufs_pm_lvl_states ufs_pm_lvl_states[] = { 238 [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 239 [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 240 [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 241 [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 242 [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 243 [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE}, 244 /* 245 * For DeepSleep, the link is first put in hibern8 and then off. 246 * Leaving the link in hibern8 is not supported. 247 */ 248 [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE}, 249}; 250 251static inline enum ufs_dev_pwr_mode 252ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl) 253{ 254 return ufs_pm_lvl_states[lvl].dev_state; 255} 256 257static inline enum uic_link_state 258ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl) 259{ 260 return ufs_pm_lvl_states[lvl].link_state; 261} 262 263static inline enum ufs_pm_level 264ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, 265 enum uic_link_state link_state) 266{ 267 enum ufs_pm_level lvl; 268 269 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) { 270 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) && 271 (ufs_pm_lvl_states[lvl].link_state == link_state)) 272 return lvl; 273 } 274 275 /* if no match found, return the level 0 */ 276 return UFS_PM_LVL_0; 277} 278 279static bool ufshcd_has_pending_tasks(struct ufs_hba *hba) 280{ 281 return hba->outstanding_tasks || hba->active_uic_cmd || 282 hba->uic_async_done; 283} 284 285static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba) 286{ 287 return scsi_host_busy(hba->host) || ufshcd_has_pending_tasks(hba); 288} 289 290static const struct ufs_dev_quirk ufs_fixups[] = { 291 /* UFS cards deviations table */ 292 { .wmanufacturerid = UFS_VENDOR_MICRON, 293 .model = UFS_ANY_MODEL, 294 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, 295 { .wmanufacturerid = UFS_VENDOR_SAMSUNG, 296 .model = UFS_ANY_MODEL, 297 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | 298 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE | 299 UFS_DEVICE_QUIRK_PA_HIBER8TIME | 300 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS }, 301 { .wmanufacturerid = UFS_VENDOR_SKHYNIX, 302 .model = UFS_ANY_MODEL, 303 .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME }, 304 { .wmanufacturerid = UFS_VENDOR_SKHYNIX, 305 .model = "hB8aL1" /*H28U62301AMR*/, 306 .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME }, 307 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 308 .model = UFS_ANY_MODEL, 309 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, 310 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 311 .model = "THGLF2G9C8KBADG", 312 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, 313 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 314 .model = "THGLF2G9D8KBADG", 315 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, 316 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 317 .model = "THGJFJT1E45BATP", 318 .quirk = UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT }, 319 {} 320}; 321 322static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba); 323static void ufshcd_async_scan(void *data, async_cookie_t cookie); 324static int ufshcd_reset_and_restore(struct ufs_hba *hba); 325static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd); 326static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); 327static void ufshcd_hba_exit(struct ufs_hba *hba); 328static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params); 329static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params); 330static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); 331static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); 332static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); 333static void ufshcd_resume_clkscaling(struct ufs_hba *hba); 334static void ufshcd_suspend_clkscaling(struct ufs_hba *hba); 335static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, 336 bool scale_up); 337static irqreturn_t ufshcd_intr(int irq, void *__hba); 338static int ufshcd_change_power_mode(struct ufs_hba *hba, 339 struct ufs_pa_layer_attr *pwr_mode); 340static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on); 341static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on); 342static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 343 struct ufs_vreg *vreg); 344static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, 345 bool enable); 346static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba); 347static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba); 348 349void ufshcd_enable_irq(struct ufs_hba *hba) 350{ 351 if (!hba->is_irq_enabled) { 352 enable_irq(hba->irq); 353 hba->is_irq_enabled = true; 354 } 355} 356EXPORT_SYMBOL_GPL(ufshcd_enable_irq); 357 358void ufshcd_disable_irq(struct ufs_hba *hba) 359{ 360 if (hba->is_irq_enabled) { 361 disable_irq(hba->irq); 362 hba->is_irq_enabled = false; 363 } 364} 365EXPORT_SYMBOL_GPL(ufshcd_disable_irq); 366 367/** 368 * ufshcd_enable_intr - enable interrupts 369 * @hba: per adapter instance 370 * @intrs: interrupt bits 371 */ 372void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) 373{ 374 u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 375 u32 new_val = old_val | intrs; 376 377 if (new_val != old_val) 378 ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); 379} 380 381/** 382 * ufshcd_disable_intr - disable interrupts 383 * @hba: per adapter instance 384 * @intrs: interrupt bits 385 */ 386static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) 387{ 388 u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 389 u32 new_val = old_val & ~intrs; 390 391 if (new_val != old_val) 392 ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); 393} 394 395static void ufshcd_configure_wb(struct ufs_hba *hba) 396{ 397 if (!ufshcd_is_wb_allowed(hba)) 398 return; 399 400 ufshcd_wb_toggle(hba, true); 401 402 ufshcd_wb_toggle_buf_flush_during_h8(hba, true); 403 404 if (ufshcd_is_wb_buf_flush_allowed(hba)) 405 ufshcd_wb_toggle_buf_flush(hba, true); 406} 407 408static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, 409 struct ufshcd_lrb *lrb, 410 enum ufs_trace_str_t str_t) 411{ 412 struct utp_upiu_req *rq = lrb->ucd_req_ptr; 413 struct utp_upiu_header *header; 414 415 if (!trace_ufshcd_upiu_enabled()) 416 return; 417 418 if (str_t == UFS_CMD_SEND) 419 header = &rq->header; 420 else 421 header = &lrb->ucd_rsp_ptr->header; 422 423 trace_ufshcd_upiu(hba, str_t, header, &rq->sc.cdb, 424 UFS_TSF_CDB); 425} 426 427static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, 428 enum ufs_trace_str_t str_t, 429 struct utp_upiu_req *rq_rsp) 430{ 431 if (!trace_ufshcd_upiu_enabled()) 432 return; 433 434 trace_ufshcd_upiu(hba, str_t, &rq_rsp->header, 435 &rq_rsp->qr, UFS_TSF_OSF); 436} 437 438static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag, 439 enum ufs_trace_str_t str_t) 440{ 441 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag]; 442 443 if (!trace_ufshcd_upiu_enabled()) 444 return; 445 446 if (str_t == UFS_TM_SEND) 447 trace_ufshcd_upiu(hba, str_t, 448 &descp->upiu_req.req_header, 449 &descp->upiu_req.input_param1, 450 UFS_TSF_TM_INPUT); 451 else 452 trace_ufshcd_upiu(hba, str_t, 453 &descp->upiu_rsp.rsp_header, 454 &descp->upiu_rsp.output_param1, 455 UFS_TSF_TM_OUTPUT); 456} 457 458static void ufshcd_add_uic_command_trace(struct ufs_hba *hba, 459 const struct uic_command *ucmd, 460 enum ufs_trace_str_t str_t) 461{ 462 u32 cmd; 463 464 if (!trace_ufshcd_uic_command_enabled()) 465 return; 466 467 if (str_t == UFS_CMD_SEND) 468 cmd = ucmd->command; 469 else 470 cmd = ufshcd_readl(hba, REG_UIC_COMMAND); 471 472 trace_ufshcd_uic_command(hba, str_t, cmd, 473 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1), 474 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2), 475 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); 476} 477 478static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd, 479 enum ufs_trace_str_t str_t) 480{ 481 u64 lba = 0; 482 u8 opcode = 0, group_id = 0; 483 u32 doorbell = 0; 484 u32 intr; 485 u32 hwq_id = 0; 486 struct request *rq = scsi_cmd_to_rq(cmd); 487 unsigned int tag = rq->tag; 488 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 489 int transfer_len = -1; 490 491 /* trace UPIU also */ 492 ufshcd_add_cmd_upiu_trace(hba, lrbp, str_t); 493 if (!trace_ufshcd_command_enabled()) 494 return; 495 496 opcode = cmd->cmnd[0]; 497 498 if (opcode == READ_10 || opcode == WRITE_10) { 499 /* 500 * Currently we only fully trace read(10) and write(10) commands 501 */ 502 transfer_len = 503 be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len); 504 lba = scsi_get_lba(cmd); 505 if (opcode == WRITE_10) 506 group_id = cmd->cmnd[6]; 507 } else if (opcode == UNMAP) { 508 /* 509 * The number of Bytes to be unmapped beginning with the lba. 510 */ 511 transfer_len = blk_rq_bytes(rq); 512 lba = scsi_get_lba(cmd); 513 } 514 515 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 516 517 if (hba->mcq_enabled) { 518 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 519 if (hwq) 520 hwq_id = hwq->id; 521 } else { 522 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 523 } 524 trace_ufshcd_command(cmd->device, hba, str_t, tag, doorbell, hwq_id, 525 transfer_len, intr, lba, opcode, group_id); 526} 527 528static void ufshcd_print_clk_freqs(struct ufs_hba *hba) 529{ 530 struct ufs_clk_info *clki; 531 struct list_head *head = &hba->clk_list_head; 532 533 if (list_empty(head)) 534 return; 535 536 list_for_each_entry(clki, head, list) { 537 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq && 538 clki->max_freq) 539 dev_err(hba->dev, "clk: %s, rate: %u\n", 540 clki->name, clki->curr_freq); 541 } 542} 543 544static void ufshcd_print_evt(struct ufs_hba *hba, u32 id, 545 const char *err_name) 546{ 547 int i; 548 bool found = false; 549 const struct ufs_event_hist *e; 550 551 if (id >= UFS_EVT_CNT) 552 return; 553 554 e = &hba->ufs_stats.event[id]; 555 556 for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) { 557 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH; 558 559 if (e->tstamp[p] == 0) 560 continue; 561 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p, 562 e->val[p], div_u64(e->tstamp[p], 1000)); 563 found = true; 564 } 565 566 if (!found) 567 dev_err(hba->dev, "No record of %s\n", err_name); 568 else 569 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt); 570} 571 572static void ufshcd_print_evt_hist(struct ufs_hba *hba) 573{ 574 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); 575 576 ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err"); 577 ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err"); 578 ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err"); 579 ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err"); 580 ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err"); 581 ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR, 582 "auto_hibern8_err"); 583 ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err"); 584 ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL, 585 "link_startup_fail"); 586 ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail"); 587 ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR, 588 "suspend_fail"); 589 ufshcd_print_evt(hba, UFS_EVT_WL_RES_ERR, "wlun resume_fail"); 590 ufshcd_print_evt(hba, UFS_EVT_WL_SUSP_ERR, 591 "wlun suspend_fail"); 592 ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset"); 593 ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset"); 594 ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort"); 595 596 ufshcd_vops_dbg_register_dump(hba); 597} 598 599static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd, 600 bool pr_prdt) 601{ 602 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 603 const int tag = scsi_cmd_to_rq(cmd)->tag; 604 int prdt_length; 605 606 if (hba->monitor.enabled) { 607 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag, 608 div_u64(lrbp->issue_time_stamp_local_clock, 1000)); 609 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", tag, 610 div_u64(lrbp->compl_time_stamp_local_clock, 1000)); 611 } 612 dev_err(hba->dev, 613 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", 614 tag, (u64)lrbp->utrd_dma_addr); 615 616 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr, 617 sizeof(struct utp_transfer_req_desc)); 618 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag, 619 (u64)lrbp->ucd_req_dma_addr); 620 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr, 621 sizeof(struct utp_upiu_req)); 622 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag, 623 (u64)lrbp->ucd_rsp_dma_addr); 624 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr, 625 sizeof(struct utp_upiu_rsp)); 626 627 prdt_length = le16_to_cpu( 628 lrbp->utr_descriptor_ptr->prd_table_length); 629 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) 630 prdt_length /= ufshcd_sg_entry_size(hba); 631 632 dev_err(hba->dev, 633 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n", 634 tag, prdt_length, 635 (u64)lrbp->ucd_prdt_dma_addr); 636 637 if (pr_prdt) 638 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr, 639 ufshcd_sg_entry_size(hba) * prdt_length); 640} 641 642static bool ufshcd_print_tr_iter(struct request *req, void *priv) 643{ 644 struct scsi_device *sdev = req->q->queuedata; 645 struct Scsi_Host *shost = sdev->host; 646 struct ufs_hba *hba = shost_priv(shost); 647 648 if (!blk_mq_is_reserved_rq(req)) 649 ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv); 650 651 return true; 652} 653 654/** 655 * ufshcd_print_trs_all - print trs for all started requests. 656 * @hba: per-adapter instance. 657 * @pr_prdt: need to print prdt or not. 658 */ 659static void ufshcd_print_trs_all(struct ufs_hba *hba, bool pr_prdt) 660{ 661 blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt); 662} 663 664static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap) 665{ 666 int tag; 667 668 for_each_set_bit(tag, &bitmap, hba->nutmrs) { 669 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag]; 670 671 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag); 672 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp)); 673 } 674} 675 676static void ufshcd_print_host_state(struct ufs_hba *hba) 677{ 678 const struct scsi_device *sdev_ufs = hba->ufs_device_wlun; 679 680 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); 681 dev_err(hba->dev, "%d outstanding reqs, tasks=0x%lx\n", 682 scsi_host_busy(hba->host), hba->outstanding_tasks); 683 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", 684 hba->saved_err, hba->saved_uic_err); 685 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", 686 hba->curr_dev_pwr_mode, hba->uic_link_state); 687 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n", 688 hba->pm_op_in_progress, hba->is_sys_suspended); 689 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n", 690 hba->auto_bkops_enabled, hba->host->host_self_blocked); 691 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state); 692 dev_err(hba->dev, 693 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n", 694 div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000), 695 hba->ufs_stats.hibern8_exit_cnt); 696 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n", 697 hba->eh_flags, hba->req_abort_count); 698 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n", 699 hba->ufs_version, hba->capabilities, hba->caps); 700 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks, 701 hba->dev_quirks); 702 if (sdev_ufs) 703 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n", 704 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev); 705 706 ufshcd_print_clk_freqs(hba); 707} 708 709/** 710 * ufshcd_print_pwr_info - print power params as saved in hba 711 * power info 712 * @hba: per-adapter instance 713 */ 714static void ufshcd_print_pwr_info(struct ufs_hba *hba) 715{ 716 static const char * const names[] = { 717 "INVALID MODE", 718 "FAST MODE", 719 "SLOW_MODE", 720 "INVALID MODE", 721 "FASTAUTO_MODE", 722 "SLOWAUTO_MODE", 723 "INVALID MODE", 724 }; 725 726 /* 727 * Using dev_dbg to avoid messages during runtime PM to avoid 728 * never-ending cycles of messages written back to storage by user space 729 * causing runtime resume, causing more messages and so on. 730 */ 731 dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n", 732 __func__, 733 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx, 734 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx, 735 names[hba->pwr_info.pwr_rx], 736 names[hba->pwr_info.pwr_tx], 737 hba->pwr_info.hs_rate); 738} 739 740static void ufshcd_device_reset(struct ufs_hba *hba) 741{ 742 int err; 743 744 err = ufshcd_vops_device_reset(hba); 745 746 if (!err) { 747 ufshcd_set_ufs_dev_active(hba); 748 if (ufshcd_is_wb_allowed(hba)) { 749 hba->dev_info.wb_enabled = false; 750 hba->dev_info.wb_buf_flush_enabled = false; 751 } 752 if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) 753 hba->dev_info.rtc_time_baseline = 0; 754 } 755 if (err != -EOPNOTSUPP) 756 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err); 757} 758 759void ufshcd_delay_us(unsigned long us, unsigned long tolerance) 760{ 761 if (!us) 762 return; 763 764 if (us < 10) 765 udelay(us); 766 else 767 usleep_range(us, us + tolerance); 768} 769EXPORT_SYMBOL_GPL(ufshcd_delay_us); 770 771/** 772 * ufshcd_wait_for_register - wait for register value to change 773 * @hba: per-adapter interface 774 * @reg: mmio register offset 775 * @mask: mask to apply to the read register value 776 * @val: value to wait for 777 * @interval_us: polling interval in microseconds 778 * @timeout_ms: timeout in milliseconds 779 * 780 * Return: -ETIMEDOUT on error, zero on success. 781 */ 782static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, 783 u32 val, unsigned long interval_us, 784 unsigned long timeout_ms) 785{ 786 u32 v; 787 788 val &= mask; /* ignore bits that we don't intend to wait on */ 789 790 return read_poll_timeout(ufshcd_readl, v, (v & mask) == val, 791 interval_us, timeout_ms * 1000, false, hba, reg); 792} 793 794/** 795 * ufshcd_get_intr_mask - Get the interrupt bit mask 796 * @hba: Pointer to adapter instance 797 * 798 * Return: interrupt bit mask per version 799 */ 800static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) 801{ 802 if (hba->ufs_version <= ufshci_version(2, 0)) 803 return INTERRUPT_MASK_ALL_VER_11; 804 805 return INTERRUPT_MASK_ALL_VER_21; 806} 807 808/** 809 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA 810 * @hba: Pointer to adapter instance 811 * 812 * Return: UFSHCI version supported by the controller 813 */ 814static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) 815{ 816 u32 ufshci_ver; 817 818 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION) 819 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba); 820 else 821 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION); 822 823 /* 824 * UFSHCI v1.x uses a different version scheme, in order 825 * to allow the use of comparisons with the ufshci_version 826 * function, we convert it to the same scheme as ufs 2.0+. 827 */ 828 if (ufshci_ver & 0x00010000) 829 return ufshci_version(1, ufshci_ver & 0x00000100); 830 831 return ufshci_ver; 832} 833 834/** 835 * ufshcd_is_device_present - Check if any device connected to 836 * the host controller 837 * @hba: pointer to adapter instance 838 * 839 * Return: true if device present, false if no device detected 840 */ 841static inline bool ufshcd_is_device_present(struct ufs_hba *hba) 842{ 843 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT; 844} 845 846/** 847 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status 848 * @lrbp: pointer to local command reference block 849 * @cqe: pointer to the completion queue entry 850 * 851 * This function is used to get the OCS field from UTRD 852 * 853 * Return: the OCS field in the UTRD. 854 */ 855static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp, 856 struct cq_entry *cqe) 857{ 858 if (cqe) 859 return cqe->overall_status & MASK_OCS; 860 861 return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS; 862} 863 864/** 865 * ufshcd_utrl_clear() - Clear requests from the controller request list. 866 * @hba: per adapter instance 867 * @mask: mask with one bit set for each request to be cleared 868 */ 869static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask) 870{ 871 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) 872 mask = ~mask; 873 /* 874 * From the UFSHCI specification: "UTP Transfer Request List CLear 875 * Register (UTRLCLR): This field is bit significant. Each bit 876 * corresponds to a slot in the UTP Transfer Request List, where bit 0 877 * corresponds to request slot 0. A bit in this field is set to ‘0’ 878 * by host software to indicate to the host controller that a transfer 879 * request slot is cleared. The host controller 880 * shall free up any resources associated to the request slot 881 * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The 882 * host software indicates no change to request slots by setting the 883 * associated bits in this field to ‘1’. Bits in this field shall only 884 * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’." 885 */ 886 ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR); 887} 888 889/** 890 * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register 891 * @hba: per adapter instance 892 * @pos: position of the bit to be cleared 893 */ 894static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos) 895{ 896 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) 897 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); 898 else 899 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); 900} 901 902/** 903 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY 904 * @reg: Register value of host controller status 905 * 906 * Return: 0 on success; a positive value if failed. 907 */ 908static inline int ufshcd_get_lists_status(u32 reg) 909{ 910 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY); 911} 912 913/** 914 * ufshcd_get_uic_cmd_result - Get the UIC command result 915 * @hba: Pointer to adapter instance 916 * 917 * This function gets the result of UIC command completion 918 * 919 * Return: 0 on success; non-zero value on error. 920 */ 921static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) 922{ 923 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) & 924 MASK_UIC_COMMAND_RESULT; 925} 926 927/** 928 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command 929 * @hba: Pointer to adapter instance 930 * 931 * This function gets UIC command argument3 932 * 933 * Return: 0 on success; non-zero value on error. 934 */ 935static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) 936{ 937 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); 938} 939 940/** 941 * ufshcd_get_req_rsp - returns the TR response transaction type 942 * @ucd_rsp_ptr: pointer to response UPIU 943 * 944 * Return: UPIU type. 945 */ 946static inline enum upiu_response_transaction 947ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) 948{ 949 return ucd_rsp_ptr->header.transaction_code; 950} 951 952/** 953 * ufshcd_is_exception_event - Check if the device raised an exception event 954 * @ucd_rsp_ptr: pointer to response UPIU 955 * 956 * The function checks if the device raised an exception event indicated in 957 * the Device Information field of response UPIU. 958 * 959 * Return: true if exception is raised, false otherwise. 960 */ 961static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) 962{ 963 return ucd_rsp_ptr->header.device_information & 1; 964} 965 966/** 967 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values. 968 * @hba: per adapter instance 969 */ 970static inline void 971ufshcd_reset_intr_aggr(struct ufs_hba *hba) 972{ 973 ufshcd_writel(hba, INT_AGGR_ENABLE | 974 INT_AGGR_COUNTER_AND_TIMER_RESET, 975 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 976} 977 978/** 979 * ufshcd_config_intr_aggr - Configure interrupt aggregation values. 980 * @hba: per adapter instance 981 * @cnt: Interrupt aggregation counter threshold 982 * @tmout: Interrupt aggregation timeout value 983 */ 984static inline void 985ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout) 986{ 987 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE | 988 INT_AGGR_COUNTER_THLD_VAL(cnt) | 989 INT_AGGR_TIMEOUT_VAL(tmout), 990 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 991} 992 993/** 994 * ufshcd_disable_intr_aggr - Disables interrupt aggregation. 995 * @hba: per adapter instance 996 */ 997static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba) 998{ 999 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 1000} 1001 1002/** 1003 * ufshcd_enable_run_stop_reg - Enable run-stop registers, 1004 * When run-stop registers are set to 1, it indicates the 1005 * host controller that it can process the requests 1006 * @hba: per adapter instance 1007 */ 1008static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba) 1009{ 1010 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT, 1011 REG_UTP_TASK_REQ_LIST_RUN_STOP); 1012 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT, 1013 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP); 1014} 1015 1016/** 1017 * ufshcd_hba_start - Start controller initialization sequence 1018 * @hba: per adapter instance 1019 */ 1020static inline void ufshcd_hba_start(struct ufs_hba *hba) 1021{ 1022 u32 val = CONTROLLER_ENABLE; 1023 1024 if (ufshcd_crypto_enable(hba)) 1025 val |= CRYPTO_GENERAL_ENABLE; 1026 1027 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE); 1028} 1029 1030/** 1031 * ufshcd_is_hba_active - Get controller state 1032 * @hba: per adapter instance 1033 * 1034 * Return: true if and only if the controller is active. 1035 */ 1036bool ufshcd_is_hba_active(struct ufs_hba *hba) 1037{ 1038 return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE; 1039} 1040EXPORT_SYMBOL_GPL(ufshcd_is_hba_active); 1041 1042/** 1043 * ufshcd_pm_qos_init - initialize PM QoS request 1044 * @hba: per adapter instance 1045 */ 1046void ufshcd_pm_qos_init(struct ufs_hba *hba) 1047{ 1048 guard(mutex)(&hba->pm_qos_mutex); 1049 1050 if (hba->pm_qos_enabled) 1051 return; 1052 1053 cpu_latency_qos_add_request(&hba->pm_qos_req, PM_QOS_DEFAULT_VALUE); 1054 1055 if (cpu_latency_qos_request_active(&hba->pm_qos_req)) 1056 hba->pm_qos_enabled = true; 1057} 1058 1059/** 1060 * ufshcd_pm_qos_exit - remove request from PM QoS 1061 * @hba: per adapter instance 1062 */ 1063void ufshcd_pm_qos_exit(struct ufs_hba *hba) 1064{ 1065 guard(mutex)(&hba->pm_qos_mutex); 1066 1067 if (!hba->pm_qos_enabled) 1068 return; 1069 1070 cpu_latency_qos_remove_request(&hba->pm_qos_req); 1071 hba->pm_qos_enabled = false; 1072} 1073 1074/** 1075 * ufshcd_pm_qos_update - update PM QoS request 1076 * @hba: per adapter instance 1077 * @on: If True, vote for perf PM QoS mode otherwise power save mode 1078 */ 1079void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on) 1080{ 1081 guard(mutex)(&hba->pm_qos_mutex); 1082 1083 if (!hba->pm_qos_enabled) 1084 return; 1085 1086 cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE); 1087} 1088EXPORT_SYMBOL_GPL(ufshcd_pm_qos_update); 1089 1090/** 1091 * ufshcd_set_clk_freq - set UFS controller clock frequencies 1092 * @hba: per adapter instance 1093 * @scale_up: If True, set max possible frequency othewise set low frequency 1094 * 1095 * Return: 0 if successful; < 0 upon failure. 1096 */ 1097static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) 1098{ 1099 int ret = 0; 1100 struct ufs_clk_info *clki; 1101 struct list_head *head = &hba->clk_list_head; 1102 1103 if (list_empty(head)) 1104 goto out; 1105 1106 list_for_each_entry(clki, head, list) { 1107 if (!IS_ERR_OR_NULL(clki->clk)) { 1108 if (scale_up && clki->max_freq) { 1109 if (clki->curr_freq == clki->max_freq) 1110 continue; 1111 1112 ret = clk_set_rate(clki->clk, clki->max_freq); 1113 if (ret) { 1114 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 1115 __func__, clki->name, 1116 clki->max_freq, ret); 1117 break; 1118 } 1119 trace_ufshcd_clk_scaling(hba, 1120 "scaled up", clki->name, 1121 clki->curr_freq, 1122 clki->max_freq); 1123 1124 clki->curr_freq = clki->max_freq; 1125 1126 } else if (!scale_up && clki->min_freq) { 1127 if (clki->curr_freq == clki->min_freq) 1128 continue; 1129 1130 ret = clk_set_rate(clki->clk, clki->min_freq); 1131 if (ret) { 1132 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 1133 __func__, clki->name, 1134 clki->min_freq, ret); 1135 break; 1136 } 1137 trace_ufshcd_clk_scaling(hba, 1138 "scaled down", clki->name, 1139 clki->curr_freq, 1140 clki->min_freq); 1141 clki->curr_freq = clki->min_freq; 1142 } 1143 } 1144 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__, 1145 clki->name, clk_get_rate(clki->clk)); 1146 } 1147 1148out: 1149 return ret; 1150} 1151 1152int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table, 1153 struct dev_pm_opp *opp, void *data, 1154 bool scaling_down) 1155{ 1156 struct ufs_hba *hba = dev_get_drvdata(dev); 1157 struct list_head *head = &hba->clk_list_head; 1158 struct ufs_clk_info *clki; 1159 unsigned long freq; 1160 u8 idx = 0; 1161 int ret; 1162 1163 list_for_each_entry(clki, head, list) { 1164 if (!IS_ERR_OR_NULL(clki->clk)) { 1165 freq = dev_pm_opp_get_freq_indexed(opp, idx++); 1166 1167 /* Do not set rate for clocks having frequency as 0 */ 1168 if (!freq) 1169 continue; 1170 1171 ret = clk_set_rate(clki->clk, freq); 1172 if (ret) { 1173 dev_err(dev, "%s: %s clk set rate(%ldHz) failed, %d\n", 1174 __func__, clki->name, freq, ret); 1175 return ret; 1176 } 1177 1178 trace_ufshcd_clk_scaling(hba, 1179 (scaling_down ? "scaled down" : "scaled up"), 1180 clki->name, hba->clk_scaling.target_freq, freq); 1181 } 1182 } 1183 1184 return 0; 1185} 1186EXPORT_SYMBOL_GPL(ufshcd_opp_config_clks); 1187 1188static int ufshcd_opp_set_rate(struct ufs_hba *hba, unsigned long freq) 1189{ 1190 struct dev_pm_opp *opp; 1191 int ret; 1192 1193 opp = dev_pm_opp_find_freq_floor_indexed(hba->dev, 1194 &freq, 0); 1195 if (IS_ERR(opp)) 1196 return PTR_ERR(opp); 1197 1198 ret = dev_pm_opp_set_opp(hba->dev, opp); 1199 dev_pm_opp_put(opp); 1200 1201 return ret; 1202} 1203 1204/** 1205 * ufshcd_scale_clks - scale up or scale down UFS controller clocks 1206 * @hba: per adapter instance 1207 * @freq: frequency to scale 1208 * @scale_up: True if scaling up and false if scaling down 1209 * 1210 * Return: 0 if successful; < 0 upon failure. 1211 */ 1212static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, 1213 bool scale_up) 1214{ 1215 int ret = 0; 1216 ktime_t start = ktime_get(); 1217 1218 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, PRE_CHANGE); 1219 if (ret) 1220 goto out; 1221 1222 if (hba->use_pm_opp) 1223 ret = ufshcd_opp_set_rate(hba, freq); 1224 else 1225 ret = ufshcd_set_clk_freq(hba, scale_up); 1226 if (ret) 1227 goto out; 1228 1229 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, POST_CHANGE); 1230 if (ret) { 1231 if (hba->use_pm_opp) 1232 ufshcd_opp_set_rate(hba, 1233 hba->devfreq->previous_freq); 1234 else 1235 ufshcd_set_clk_freq(hba, !scale_up); 1236 goto out; 1237 } 1238 1239 ufshcd_pm_qos_update(hba, scale_up); 1240 1241out: 1242 trace_ufshcd_profile_clk_scaling(hba, 1243 (scale_up ? "up" : "down"), 1244 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 1245 return ret; 1246} 1247 1248/** 1249 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not 1250 * @hba: per adapter instance 1251 * @freq: frequency to scale 1252 * @scale_up: True if scaling up and false if scaling down 1253 * 1254 * Return: true if scaling is required, false otherwise. 1255 */ 1256static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, 1257 unsigned long freq, bool scale_up) 1258{ 1259 struct ufs_clk_info *clki; 1260 struct list_head *head = &hba->clk_list_head; 1261 1262 if (list_empty(head)) 1263 return false; 1264 1265 if (hba->use_pm_opp) 1266 return freq != hba->clk_scaling.target_freq; 1267 1268 list_for_each_entry(clki, head, list) { 1269 if (!IS_ERR_OR_NULL(clki->clk)) { 1270 if (scale_up && clki->max_freq) { 1271 if (clki->curr_freq == clki->max_freq) 1272 continue; 1273 return true; 1274 } else if (!scale_up && clki->min_freq) { 1275 if (clki->curr_freq == clki->min_freq) 1276 continue; 1277 return true; 1278 } 1279 } 1280 } 1281 1282 return false; 1283} 1284 1285/* 1286 * Determine the number of pending commands by counting the bits in the SCSI 1287 * device budget maps. This approach has been selected because a bit is set in 1288 * the budget map before scsi_host_queue_ready() checks the host_self_blocked 1289 * flag. The host_self_blocked flag can be modified by calling 1290 * scsi_block_requests() or scsi_unblock_requests(). 1291 */ 1292static u32 ufshcd_pending_cmds(struct ufs_hba *hba) 1293{ 1294 struct scsi_device *sdev; 1295 unsigned long flags; 1296 u32 pending = 0; 1297 1298 spin_lock_irqsave(hba->host->host_lock, flags); 1299 __shost_for_each_device(sdev, hba->host) 1300 pending += scsi_device_busy(sdev); 1301 spin_unlock_irqrestore(hba->host->host_lock, flags); 1302 1303 return pending; 1304} 1305 1306/* 1307 * Wait until all pending SCSI commands and TMFs have finished or the timeout 1308 * has expired. 1309 * 1310 * Return: 0 upon success; -EBUSY upon timeout. 1311 */ 1312static int ufshcd_wait_for_pending_cmds(struct ufs_hba *hba, 1313 u64 wait_timeout_us) 1314{ 1315 int ret = 0; 1316 u32 tm_doorbell; 1317 u32 tr_pending; 1318 bool timeout = false, do_last_check = false; 1319 ktime_t start; 1320 1321 ufshcd_hold(hba); 1322 /* 1323 * Wait for all the outstanding tasks/transfer requests. 1324 * Verify by checking the doorbell registers are clear. 1325 */ 1326 start = ktime_get(); 1327 do { 1328 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) { 1329 ret = -EBUSY; 1330 goto out; 1331 } 1332 1333 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 1334 tr_pending = ufshcd_pending_cmds(hba); 1335 if (!tm_doorbell && !tr_pending) { 1336 timeout = false; 1337 break; 1338 } else if (do_last_check) { 1339 break; 1340 } 1341 1342 io_schedule_timeout(msecs_to_jiffies(20)); 1343 if (ktime_to_us(ktime_sub(ktime_get(), start)) > 1344 wait_timeout_us) { 1345 timeout = true; 1346 /* 1347 * We might have scheduled out for long time so make 1348 * sure to check if doorbells are cleared by this time 1349 * or not. 1350 */ 1351 do_last_check = true; 1352 } 1353 } while (tm_doorbell || tr_pending); 1354 1355 if (timeout) { 1356 dev_err(hba->dev, 1357 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n", 1358 __func__, tm_doorbell, tr_pending); 1359 ret = -EBUSY; 1360 } 1361out: 1362 ufshcd_release(hba); 1363 return ret; 1364} 1365 1366/** 1367 * ufshcd_scale_gear - scale up/down UFS gear 1368 * @hba: per adapter instance 1369 * @target_gear: target gear to scale to 1370 * @scale_up: True for scaling up gear and false for scaling down 1371 * 1372 * Return: 0 for success; -EBUSY if scaling can't happen at this time; 1373 * non-zero for any other errors. 1374 */ 1375static int ufshcd_scale_gear(struct ufs_hba *hba, u32 target_gear, bool scale_up) 1376{ 1377 int ret = 0; 1378 struct ufs_pa_layer_attr new_pwr_info; 1379 1380 if (target_gear) { 1381 new_pwr_info = hba->pwr_info; 1382 new_pwr_info.gear_tx = target_gear; 1383 new_pwr_info.gear_rx = target_gear; 1384 1385 goto config_pwr_mode; 1386 } 1387 1388 /* Legacy gear scaling, in case vops_freq_to_gear_speed() is not implemented */ 1389 if (scale_up) { 1390 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info, 1391 sizeof(struct ufs_pa_layer_attr)); 1392 } else { 1393 memcpy(&new_pwr_info, &hba->pwr_info, 1394 sizeof(struct ufs_pa_layer_attr)); 1395 1396 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear || 1397 hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) { 1398 /* save the current power mode */ 1399 memcpy(&hba->clk_scaling.saved_pwr_info, 1400 &hba->pwr_info, 1401 sizeof(struct ufs_pa_layer_attr)); 1402 1403 /* scale down gear */ 1404 new_pwr_info.gear_tx = hba->clk_scaling.min_gear; 1405 new_pwr_info.gear_rx = hba->clk_scaling.min_gear; 1406 } 1407 } 1408 1409config_pwr_mode: 1410 /* check if the power mode needs to be changed or not? */ 1411 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info); 1412 if (ret) 1413 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)", 1414 __func__, ret, 1415 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx, 1416 new_pwr_info.gear_tx, new_pwr_info.gear_rx); 1417 1418 return ret; 1419} 1420 1421/* 1422 * Wait until all pending SCSI commands and TMFs have finished or the timeout 1423 * has expired. 1424 * 1425 * Return: 0 upon success; -EBUSY upon timeout. 1426 */ 1427static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us) 1428{ 1429 int ret = 0; 1430 /* 1431 * make sure that there are no outstanding requests when 1432 * clock scaling is in progress 1433 */ 1434 mutex_lock(&hba->host->scan_mutex); 1435 blk_mq_quiesce_tagset(&hba->host->tag_set); 1436 mutex_lock(&hba->wb_mutex); 1437 down_write(&hba->clk_scaling_lock); 1438 1439 if (!hba->clk_scaling.is_allowed || 1440 ufshcd_wait_for_pending_cmds(hba, timeout_us)) { 1441 ret = -EBUSY; 1442 up_write(&hba->clk_scaling_lock); 1443 mutex_unlock(&hba->wb_mutex); 1444 blk_mq_unquiesce_tagset(&hba->host->tag_set); 1445 mutex_unlock(&hba->host->scan_mutex); 1446 goto out; 1447 } 1448 1449 /* let's not get into low power until clock scaling is completed */ 1450 ufshcd_hold(hba); 1451 1452out: 1453 return ret; 1454} 1455 1456static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err) 1457{ 1458 up_write(&hba->clk_scaling_lock); 1459 mutex_unlock(&hba->wb_mutex); 1460 blk_mq_unquiesce_tagset(&hba->host->tag_set); 1461 mutex_unlock(&hba->host->scan_mutex); 1462 1463 /* Enable Write Booster if current gear requires it else disable it */ 1464 if (ufshcd_enable_wb_if_scaling_up(hba) && !err) 1465 ufshcd_wb_toggle(hba, hba->pwr_info.gear_rx >= hba->clk_scaling.wb_gear); 1466 1467 ufshcd_release(hba); 1468} 1469 1470/** 1471 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear 1472 * @hba: per adapter instance 1473 * @freq: frequency to scale 1474 * @scale_up: True for scaling up and false for scalin down 1475 * 1476 * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero 1477 * for any other errors. 1478 */ 1479static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq, 1480 bool scale_up) 1481{ 1482 u32 old_gear = hba->pwr_info.gear_rx; 1483 u32 new_gear = 0; 1484 int ret = 0; 1485 1486 new_gear = ufshcd_vops_freq_to_gear_speed(hba, freq); 1487 1488 ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC); 1489 if (ret) 1490 return ret; 1491 1492 /* scale down the gear before scaling down clocks */ 1493 if (!scale_up) { 1494 ret = ufshcd_scale_gear(hba, new_gear, false); 1495 if (ret) 1496 goto out_unprepare; 1497 } 1498 1499 ret = ufshcd_scale_clks(hba, freq, scale_up); 1500 if (ret) { 1501 if (!scale_up) 1502 ufshcd_scale_gear(hba, old_gear, true); 1503 goto out_unprepare; 1504 } 1505 1506 /* scale up the gear after scaling up clocks */ 1507 if (scale_up) { 1508 ret = ufshcd_scale_gear(hba, new_gear, true); 1509 if (ret) { 1510 ufshcd_scale_clks(hba, hba->devfreq->previous_freq, 1511 false); 1512 goto out_unprepare; 1513 } 1514 } 1515 1516out_unprepare: 1517 ufshcd_clock_scaling_unprepare(hba, ret); 1518 return ret; 1519} 1520 1521static void ufshcd_clk_scaling_suspend_work(struct work_struct *work) 1522{ 1523 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1524 clk_scaling.suspend_work); 1525 1526 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1527 { 1528 if (hba->clk_scaling.active_reqs || 1529 hba->clk_scaling.is_suspended) 1530 return; 1531 1532 hba->clk_scaling.is_suspended = true; 1533 hba->clk_scaling.window_start_t = 0; 1534 } 1535 1536 devfreq_suspend_device(hba->devfreq); 1537} 1538 1539static void ufshcd_clk_scaling_resume_work(struct work_struct *work) 1540{ 1541 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1542 clk_scaling.resume_work); 1543 1544 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1545 { 1546 if (!hba->clk_scaling.is_suspended) 1547 return; 1548 hba->clk_scaling.is_suspended = false; 1549 } 1550 1551 devfreq_resume_device(hba->devfreq); 1552} 1553 1554static int ufshcd_devfreq_target(struct device *dev, 1555 unsigned long *freq, u32 flags) 1556{ 1557 int ret = 0; 1558 struct ufs_hba *hba = dev_get_drvdata(dev); 1559 ktime_t start; 1560 bool scale_up = false, sched_clk_scaling_suspend_work = false; 1561 struct list_head *clk_list = &hba->clk_list_head; 1562 struct ufs_clk_info *clki; 1563 1564 if (!ufshcd_is_clkscaling_supported(hba)) 1565 return -EINVAL; 1566 1567 if (hba->use_pm_opp) { 1568 struct dev_pm_opp *opp; 1569 1570 /* Get the recommended frequency from OPP framework */ 1571 opp = devfreq_recommended_opp(dev, freq, flags); 1572 if (IS_ERR(opp)) 1573 return PTR_ERR(opp); 1574 1575 dev_pm_opp_put(opp); 1576 } else { 1577 /* Override with the closest supported frequency */ 1578 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, 1579 list); 1580 *freq = (unsigned long) clk_round_rate(clki->clk, *freq); 1581 } 1582 1583 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1584 { 1585 if (ufshcd_eh_in_progress(hba)) 1586 return 0; 1587 1588 /* Skip scaling clock when clock scaling is suspended */ 1589 if (hba->clk_scaling.is_suspended) { 1590 dev_warn(hba->dev, "clock scaling is suspended, skip"); 1591 return 0; 1592 } 1593 1594 if (!hba->clk_scaling.active_reqs) 1595 sched_clk_scaling_suspend_work = true; 1596 1597 if (list_empty(clk_list)) 1598 goto out; 1599 1600 /* Decide based on the target or rounded-off frequency and update */ 1601 if (hba->use_pm_opp) 1602 scale_up = *freq > hba->clk_scaling.target_freq; 1603 else 1604 scale_up = *freq == clki->max_freq; 1605 1606 if (!hba->use_pm_opp && !scale_up) 1607 *freq = clki->min_freq; 1608 1609 /* Update the frequency */ 1610 if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) { 1611 ret = 0; 1612 goto out; /* no state change required */ 1613 } 1614 } 1615 1616 start = ktime_get(); 1617 ret = ufshcd_devfreq_scale(hba, *freq, scale_up); 1618 if (!ret) 1619 hba->clk_scaling.target_freq = *freq; 1620 1621 trace_ufshcd_profile_clk_scaling(hba, 1622 (scale_up ? "up" : "down"), 1623 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 1624 1625out: 1626 if (sched_clk_scaling_suspend_work && 1627 (!scale_up || hba->clk_scaling.suspend_on_no_request)) 1628 queue_work(hba->clk_scaling.workq, 1629 &hba->clk_scaling.suspend_work); 1630 1631 return ret; 1632} 1633 1634static int ufshcd_devfreq_get_dev_status(struct device *dev, 1635 struct devfreq_dev_status *stat) 1636{ 1637 struct ufs_hba *hba = dev_get_drvdata(dev); 1638 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 1639 ktime_t curr_t; 1640 1641 if (!ufshcd_is_clkscaling_supported(hba)) 1642 return -EINVAL; 1643 1644 memset(stat, 0, sizeof(*stat)); 1645 1646 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 1647 1648 curr_t = ktime_get(); 1649 if (!scaling->window_start_t) 1650 goto start_window; 1651 1652 /* 1653 * If current frequency is 0, then the ondemand governor considers 1654 * there's no initial frequency set. And it always requests to set 1655 * to max. frequency. 1656 */ 1657 if (hba->use_pm_opp) { 1658 stat->current_frequency = hba->clk_scaling.target_freq; 1659 } else { 1660 struct list_head *clk_list = &hba->clk_list_head; 1661 struct ufs_clk_info *clki; 1662 1663 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1664 stat->current_frequency = clki->curr_freq; 1665 } 1666 1667 if (scaling->is_busy_started) 1668 scaling->tot_busy_t += ktime_us_delta(curr_t, 1669 scaling->busy_start_t); 1670 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t); 1671 stat->busy_time = scaling->tot_busy_t; 1672start_window: 1673 scaling->window_start_t = curr_t; 1674 scaling->tot_busy_t = 0; 1675 1676 if (scaling->active_reqs) { 1677 scaling->busy_start_t = curr_t; 1678 scaling->is_busy_started = true; 1679 } else { 1680 scaling->busy_start_t = 0; 1681 scaling->is_busy_started = false; 1682 } 1683 1684 return 0; 1685} 1686 1687static int ufshcd_devfreq_init(struct ufs_hba *hba) 1688{ 1689 struct list_head *clk_list = &hba->clk_list_head; 1690 struct ufs_clk_info *clki; 1691 struct devfreq *devfreq; 1692 int ret; 1693 1694 /* Skip devfreq if we don't have any clocks in the list */ 1695 if (list_empty(clk_list)) 1696 return 0; 1697 1698 if (!hba->use_pm_opp) { 1699 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1700 dev_pm_opp_add(hba->dev, clki->min_freq, 0); 1701 dev_pm_opp_add(hba->dev, clki->max_freq, 0); 1702 } 1703 1704 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile, 1705 &hba->vps->ondemand_data); 1706 devfreq = devfreq_add_device(hba->dev, 1707 &hba->vps->devfreq_profile, 1708 DEVFREQ_GOV_SIMPLE_ONDEMAND, 1709 &hba->vps->ondemand_data); 1710 if (IS_ERR(devfreq)) { 1711 ret = PTR_ERR(devfreq); 1712 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret); 1713 1714 if (!hba->use_pm_opp) { 1715 dev_pm_opp_remove(hba->dev, clki->min_freq); 1716 dev_pm_opp_remove(hba->dev, clki->max_freq); 1717 } 1718 return ret; 1719 } 1720 1721 hba->devfreq = devfreq; 1722 1723 return 0; 1724} 1725 1726static void ufshcd_devfreq_remove(struct ufs_hba *hba) 1727{ 1728 struct list_head *clk_list = &hba->clk_list_head; 1729 1730 if (!hba->devfreq) 1731 return; 1732 1733 devfreq_remove_device(hba->devfreq); 1734 hba->devfreq = NULL; 1735 1736 if (!hba->use_pm_opp) { 1737 struct ufs_clk_info *clki; 1738 1739 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1740 dev_pm_opp_remove(hba->dev, clki->min_freq); 1741 dev_pm_opp_remove(hba->dev, clki->max_freq); 1742 } 1743} 1744 1745static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) 1746{ 1747 bool suspend = false; 1748 1749 cancel_work_sync(&hba->clk_scaling.suspend_work); 1750 cancel_work_sync(&hba->clk_scaling.resume_work); 1751 1752 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1753 { 1754 if (!hba->clk_scaling.is_suspended) { 1755 suspend = true; 1756 hba->clk_scaling.is_suspended = true; 1757 hba->clk_scaling.window_start_t = 0; 1758 } 1759 } 1760 1761 if (suspend) 1762 devfreq_suspend_device(hba->devfreq); 1763} 1764 1765static void ufshcd_resume_clkscaling(struct ufs_hba *hba) 1766{ 1767 bool resume = false; 1768 1769 scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) 1770 { 1771 if (hba->clk_scaling.is_suspended) { 1772 resume = true; 1773 hba->clk_scaling.is_suspended = false; 1774 } 1775 } 1776 1777 if (resume) 1778 devfreq_resume_device(hba->devfreq); 1779} 1780 1781static ssize_t ufshcd_clkscale_enable_show(struct device *dev, 1782 struct device_attribute *attr, char *buf) 1783{ 1784 struct ufs_hba *hba = dev_get_drvdata(dev); 1785 1786 return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled); 1787} 1788 1789static ssize_t ufshcd_clkscale_enable_store(struct device *dev, 1790 struct device_attribute *attr, const char *buf, size_t count) 1791{ 1792 struct ufs_hba *hba = dev_get_drvdata(dev); 1793 struct ufs_clk_info *clki; 1794 unsigned long freq; 1795 u32 value; 1796 int err = 0; 1797 1798 if (kstrtou32(buf, 0, &value)) 1799 return -EINVAL; 1800 1801 down(&hba->host_sem); 1802 if (!ufshcd_is_user_access_allowed(hba)) { 1803 err = -EBUSY; 1804 goto out; 1805 } 1806 1807 value = !!value; 1808 if (value == hba->clk_scaling.is_enabled) 1809 goto out; 1810 1811 ufshcd_rpm_get_sync(hba); 1812 ufshcd_hold(hba); 1813 1814 hba->clk_scaling.is_enabled = value; 1815 1816 if (value) { 1817 ufshcd_resume_clkscaling(hba); 1818 goto out_rel; 1819 } 1820 1821 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list); 1822 freq = clki->max_freq; 1823 1824 ufshcd_suspend_clkscaling(hba); 1825 1826 if (!ufshcd_is_devfreq_scaling_required(hba, freq, true)) 1827 goto out_rel; 1828 1829 err = ufshcd_devfreq_scale(hba, freq, true); 1830 if (err) 1831 dev_err(hba->dev, "%s: failed to scale clocks up %d\n", 1832 __func__, err); 1833 else 1834 hba->clk_scaling.target_freq = freq; 1835 1836out_rel: 1837 ufshcd_release(hba); 1838 ufshcd_rpm_put_sync(hba); 1839out: 1840 up(&hba->host_sem); 1841 return err ? err : count; 1842} 1843 1844static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba) 1845{ 1846 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show; 1847 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store; 1848 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr); 1849 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable"; 1850 hba->clk_scaling.enable_attr.attr.mode = 0644; 1851 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr)) 1852 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n"); 1853} 1854 1855static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba) 1856{ 1857 if (hba->clk_scaling.enable_attr.attr.name) 1858 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr); 1859} 1860 1861static void ufshcd_init_clk_scaling(struct ufs_hba *hba) 1862{ 1863 if (!ufshcd_is_clkscaling_supported(hba)) 1864 return; 1865 1866 if (!hba->clk_scaling.min_gear) 1867 hba->clk_scaling.min_gear = UFS_HS_G1; 1868 1869 if (!hba->clk_scaling.wb_gear) 1870 /* Use intermediate gear speed HS_G3 as the default wb_gear */ 1871 hba->clk_scaling.wb_gear = UFS_HS_G3; 1872 1873 INIT_WORK(&hba->clk_scaling.suspend_work, 1874 ufshcd_clk_scaling_suspend_work); 1875 INIT_WORK(&hba->clk_scaling.resume_work, 1876 ufshcd_clk_scaling_resume_work); 1877 1878 spin_lock_init(&hba->clk_scaling.lock); 1879 1880 hba->clk_scaling.workq = alloc_ordered_workqueue( 1881 "ufs_clkscaling_%d", WQ_MEM_RECLAIM, hba->host->host_no); 1882 1883 hba->clk_scaling.is_initialized = true; 1884} 1885 1886static void ufshcd_exit_clk_scaling(struct ufs_hba *hba) 1887{ 1888 if (!hba->clk_scaling.is_initialized) 1889 return; 1890 1891 ufshcd_remove_clk_scaling_sysfs(hba); 1892 destroy_workqueue(hba->clk_scaling.workq); 1893 ufshcd_devfreq_remove(hba); 1894 hba->clk_scaling.is_initialized = false; 1895} 1896 1897static void ufshcd_ungate_work(struct work_struct *work) 1898{ 1899 int ret; 1900 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1901 clk_gating.ungate_work); 1902 1903 cancel_delayed_work_sync(&hba->clk_gating.gate_work); 1904 1905 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { 1906 if (hba->clk_gating.state == CLKS_ON) 1907 return; 1908 } 1909 1910 ufshcd_hba_vreg_set_hpm(hba); 1911 ufshcd_setup_clocks(hba, true); 1912 1913 ufshcd_enable_irq(hba); 1914 1915 /* Exit from hibern8 */ 1916 if (ufshcd_can_hibern8_during_gating(hba)) { 1917 /* Prevent gating in this path */ 1918 hba->clk_gating.is_suspended = true; 1919 if (ufshcd_is_link_hibern8(hba)) { 1920 ret = ufshcd_uic_hibern8_exit(hba); 1921 if (ret) 1922 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 1923 __func__, ret); 1924 else 1925 ufshcd_set_link_active(hba); 1926 } 1927 hba->clk_gating.is_suspended = false; 1928 } 1929} 1930 1931/** 1932 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release. 1933 * Also, exit from hibern8 mode and set the link as active. 1934 * @hba: per adapter instance 1935 */ 1936void ufshcd_hold(struct ufs_hba *hba) 1937{ 1938 bool flush_result; 1939 unsigned long flags; 1940 1941 if (!ufshcd_is_clkgating_allowed(hba) || 1942 !hba->clk_gating.is_initialized) 1943 return; 1944 spin_lock_irqsave(&hba->clk_gating.lock, flags); 1945 hba->clk_gating.active_reqs++; 1946 1947start: 1948 switch (hba->clk_gating.state) { 1949 case CLKS_ON: 1950 /* 1951 * Wait for the ungate work to complete if in progress. 1952 * Though the clocks may be in ON state, the link could 1953 * still be in hibner8 state if hibern8 is allowed 1954 * during clock gating. 1955 * Make sure we exit hibern8 state also in addition to 1956 * clocks being ON. 1957 */ 1958 if (ufshcd_can_hibern8_during_gating(hba) && 1959 ufshcd_is_link_hibern8(hba)) { 1960 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 1961 flush_result = flush_work(&hba->clk_gating.ungate_work); 1962 if (hba->clk_gating.is_suspended && !flush_result) 1963 return; 1964 spin_lock_irqsave(&hba->clk_gating.lock, flags); 1965 goto start; 1966 } 1967 break; 1968 case REQ_CLKS_OFF: 1969 if (cancel_delayed_work(&hba->clk_gating.gate_work)) { 1970 hba->clk_gating.state = CLKS_ON; 1971 trace_ufshcd_clk_gating(hba, 1972 hba->clk_gating.state); 1973 break; 1974 } 1975 /* 1976 * If we are here, it means gating work is either done or 1977 * currently running. Hence, fall through to cancel gating 1978 * work and to enable clocks. 1979 */ 1980 fallthrough; 1981 case CLKS_OFF: 1982 hba->clk_gating.state = REQ_CLKS_ON; 1983 trace_ufshcd_clk_gating(hba, 1984 hba->clk_gating.state); 1985 queue_work(hba->clk_gating.clk_gating_workq, 1986 &hba->clk_gating.ungate_work); 1987 /* 1988 * fall through to check if we should wait for this 1989 * work to be done or not. 1990 */ 1991 fallthrough; 1992 case REQ_CLKS_ON: 1993 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 1994 flush_work(&hba->clk_gating.ungate_work); 1995 /* Make sure state is CLKS_ON before returning */ 1996 spin_lock_irqsave(&hba->clk_gating.lock, flags); 1997 goto start; 1998 default: 1999 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n", 2000 __func__, hba->clk_gating.state); 2001 break; 2002 } 2003 spin_unlock_irqrestore(&hba->clk_gating.lock, flags); 2004} 2005EXPORT_SYMBOL_GPL(ufshcd_hold); 2006 2007static void ufshcd_gate_work(struct work_struct *work) 2008{ 2009 struct ufs_hba *hba = container_of(work, struct ufs_hba, 2010 clk_gating.gate_work.work); 2011 int ret; 2012 2013 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { 2014 /* 2015 * In case you are here to cancel this work the gating state 2016 * would be marked as REQ_CLKS_ON. In this case save time by 2017 * skipping the gating work and exit after changing the clock 2018 * state to CLKS_ON. 2019 */ 2020 if (hba->clk_gating.is_suspended || 2021 hba->clk_gating.state != REQ_CLKS_OFF) { 2022 hba->clk_gating.state = CLKS_ON; 2023 trace_ufshcd_clk_gating(hba, 2024 hba->clk_gating.state); 2025 return; 2026 } 2027 2028 if (hba->clk_gating.active_reqs) 2029 return; 2030 } 2031 2032 scoped_guard(spinlock_irqsave, hba->host->host_lock) { 2033 if (ufshcd_is_ufs_dev_busy(hba) || 2034 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) 2035 return; 2036 } 2037 2038 /* put the link into hibern8 mode before turning off clocks */ 2039 if (ufshcd_can_hibern8_during_gating(hba)) { 2040 ret = ufshcd_uic_hibern8_enter(hba); 2041 if (ret) { 2042 hba->clk_gating.state = CLKS_ON; 2043 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 2044 __func__, ret); 2045 trace_ufshcd_clk_gating(hba, 2046 hba->clk_gating.state); 2047 return; 2048 } 2049 ufshcd_set_link_hibern8(hba); 2050 } 2051 2052 ufshcd_disable_irq(hba); 2053 2054 ufshcd_setup_clocks(hba, false); 2055 2056 /* Put the host controller in low power mode if possible */ 2057 ufshcd_hba_vreg_set_lpm(hba); 2058 /* 2059 * In case you are here to cancel this work the gating state 2060 * would be marked as REQ_CLKS_ON. In this case keep the state 2061 * as REQ_CLKS_ON which would anyway imply that clocks are off 2062 * and a request to turn them on is pending. By doing this way, 2063 * we keep the state machine in tact and this would ultimately 2064 * prevent from doing cancel work multiple times when there are 2065 * new requests arriving before the current cancel work is done. 2066 */ 2067 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2068 if (hba->clk_gating.state == REQ_CLKS_OFF) { 2069 hba->clk_gating.state = CLKS_OFF; 2070 trace_ufshcd_clk_gating(hba, 2071 hba->clk_gating.state); 2072 } 2073} 2074 2075static void __ufshcd_release(struct ufs_hba *hba) 2076{ 2077 lockdep_assert_held(&hba->clk_gating.lock); 2078 2079 if (!ufshcd_is_clkgating_allowed(hba)) 2080 return; 2081 2082 hba->clk_gating.active_reqs--; 2083 2084 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || 2085 !hba->clk_gating.is_initialized || 2086 hba->clk_gating.state == CLKS_OFF) 2087 return; 2088 2089 scoped_guard(spinlock_irqsave, hba->host->host_lock) { 2090 if (ufshcd_has_pending_tasks(hba) || 2091 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) 2092 return; 2093 } 2094 2095 hba->clk_gating.state = REQ_CLKS_OFF; 2096 trace_ufshcd_clk_gating(hba, hba->clk_gating.state); 2097 queue_delayed_work(hba->clk_gating.clk_gating_workq, 2098 &hba->clk_gating.gate_work, 2099 msecs_to_jiffies(hba->clk_gating.delay_ms)); 2100} 2101 2102void ufshcd_release(struct ufs_hba *hba) 2103{ 2104 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2105 __ufshcd_release(hba); 2106} 2107EXPORT_SYMBOL_GPL(ufshcd_release); 2108 2109static ssize_t ufshcd_clkgate_delay_show(struct device *dev, 2110 struct device_attribute *attr, char *buf) 2111{ 2112 struct ufs_hba *hba = dev_get_drvdata(dev); 2113 2114 return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms); 2115} 2116 2117void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value) 2118{ 2119 struct ufs_hba *hba = dev_get_drvdata(dev); 2120 2121 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2122 hba->clk_gating.delay_ms = value; 2123} 2124EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set); 2125 2126static ssize_t ufshcd_clkgate_delay_store(struct device *dev, 2127 struct device_attribute *attr, const char *buf, size_t count) 2128{ 2129 unsigned long value; 2130 2131 if (kstrtoul(buf, 0, &value)) 2132 return -EINVAL; 2133 2134 ufshcd_clkgate_delay_set(dev, value); 2135 return count; 2136} 2137 2138static ssize_t ufshcd_clkgate_enable_show(struct device *dev, 2139 struct device_attribute *attr, char *buf) 2140{ 2141 struct ufs_hba *hba = dev_get_drvdata(dev); 2142 2143 return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled); 2144} 2145 2146static ssize_t ufshcd_clkgate_enable_store(struct device *dev, 2147 struct device_attribute *attr, const char *buf, size_t count) 2148{ 2149 struct ufs_hba *hba = dev_get_drvdata(dev); 2150 u32 value; 2151 2152 if (kstrtou32(buf, 0, &value)) 2153 return -EINVAL; 2154 2155 value = !!value; 2156 2157 guard(spinlock_irqsave)(&hba->clk_gating.lock); 2158 2159 if (value == hba->clk_gating.is_enabled) 2160 return count; 2161 2162 if (value) 2163 __ufshcd_release(hba); 2164 else 2165 hba->clk_gating.active_reqs++; 2166 2167 hba->clk_gating.is_enabled = value; 2168 2169 return count; 2170} 2171 2172static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba) 2173{ 2174 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show; 2175 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store; 2176 sysfs_attr_init(&hba->clk_gating.delay_attr.attr); 2177 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms"; 2178 hba->clk_gating.delay_attr.attr.mode = 0644; 2179 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr)) 2180 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n"); 2181 2182 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show; 2183 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store; 2184 sysfs_attr_init(&hba->clk_gating.enable_attr.attr); 2185 hba->clk_gating.enable_attr.attr.name = "clkgate_enable"; 2186 hba->clk_gating.enable_attr.attr.mode = 0644; 2187 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr)) 2188 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n"); 2189} 2190 2191static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba) 2192{ 2193 if (hba->clk_gating.delay_attr.attr.name) 2194 device_remove_file(hba->dev, &hba->clk_gating.delay_attr); 2195 if (hba->clk_gating.enable_attr.attr.name) 2196 device_remove_file(hba->dev, &hba->clk_gating.enable_attr); 2197} 2198 2199static void ufshcd_init_clk_gating(struct ufs_hba *hba) 2200{ 2201 if (!ufshcd_is_clkgating_allowed(hba)) 2202 return; 2203 2204 hba->clk_gating.state = CLKS_ON; 2205 2206 hba->clk_gating.delay_ms = 150; 2207 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work); 2208 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work); 2209 2210 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue( 2211 "ufs_clk_gating_%d", WQ_MEM_RECLAIM | WQ_HIGHPRI, 2212 hba->host->host_no); 2213 2214 ufshcd_init_clk_gating_sysfs(hba); 2215 2216 hba->clk_gating.is_enabled = true; 2217 hba->clk_gating.is_initialized = true; 2218} 2219 2220static void ufshcd_exit_clk_gating(struct ufs_hba *hba) 2221{ 2222 if (!hba->clk_gating.is_initialized) 2223 return; 2224 2225 ufshcd_remove_clk_gating_sysfs(hba); 2226 2227 /* Ungate the clock if necessary. */ 2228 ufshcd_hold(hba); 2229 hba->clk_gating.is_initialized = false; 2230 ufshcd_release(hba); 2231 2232 destroy_workqueue(hba->clk_gating.clk_gating_workq); 2233} 2234 2235static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) 2236{ 2237 bool queue_resume_work = false; 2238 ktime_t curr_t; 2239 2240 if (!ufshcd_is_clkscaling_supported(hba)) 2241 return; 2242 2243 curr_t = ktime_get(); 2244 2245 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 2246 2247 if (!hba->clk_scaling.active_reqs++) 2248 queue_resume_work = true; 2249 2250 if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) 2251 return; 2252 2253 if (queue_resume_work) 2254 queue_work(hba->clk_scaling.workq, 2255 &hba->clk_scaling.resume_work); 2256 2257 if (!hba->clk_scaling.window_start_t) { 2258 hba->clk_scaling.window_start_t = curr_t; 2259 hba->clk_scaling.tot_busy_t = 0; 2260 hba->clk_scaling.is_busy_started = false; 2261 } 2262 2263 if (!hba->clk_scaling.is_busy_started) { 2264 hba->clk_scaling.busy_start_t = curr_t; 2265 hba->clk_scaling.is_busy_started = true; 2266 } 2267} 2268 2269static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) 2270{ 2271 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 2272 2273 if (!ufshcd_is_clkscaling_supported(hba)) 2274 return; 2275 2276 guard(spinlock_irqsave)(&hba->clk_scaling.lock); 2277 2278 hba->clk_scaling.active_reqs--; 2279 if (!scaling->active_reqs && scaling->is_busy_started) { 2280 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), 2281 scaling->busy_start_t)); 2282 scaling->busy_start_t = 0; 2283 scaling->is_busy_started = false; 2284 } 2285} 2286 2287static inline int ufshcd_monitor_opcode2dir(u8 opcode) 2288{ 2289 if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16) 2290 return READ; 2291 else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16) 2292 return WRITE; 2293 else 2294 return -EINVAL; 2295} 2296 2297/* Must only be called for SCSI commands. */ 2298static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, 2299 struct scsi_cmnd *cmd) 2300{ 2301 const struct ufs_hba_monitor *m = &hba->monitor; 2302 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2303 2304 return m->enabled && 2305 (!m->chunk_size || m->chunk_size == cmd->sdb.length) && 2306 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp); 2307} 2308 2309static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2310{ 2311 int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); 2312 unsigned long flags; 2313 2314 spin_lock_irqsave(hba->host->host_lock, flags); 2315 if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0) 2316 hba->monitor.busy_start_ts[dir] = ktime_get(); 2317 spin_unlock_irqrestore(hba->host->host_lock, flags); 2318} 2319 2320static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2321{ 2322 struct request *req = scsi_cmd_to_rq(cmd); 2323 const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2324 int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); 2325 unsigned long flags; 2326 2327 spin_lock_irqsave(hba->host->host_lock, flags); 2328 if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) { 2329 struct ufs_hba_monitor *m = &hba->monitor; 2330 ktime_t now, inc, lat; 2331 2332 now = lrbp->compl_time_stamp; 2333 inc = ktime_sub(now, m->busy_start_ts[dir]); 2334 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc); 2335 m->nr_sec_rw[dir] += blk_rq_sectors(req); 2336 2337 /* Update latencies */ 2338 m->nr_req[dir]++; 2339 lat = ktime_sub(now, lrbp->issue_time_stamp); 2340 m->lat_sum[dir] += lat; 2341 if (m->lat_max[dir] < lat || !m->lat_max[dir]) 2342 m->lat_max[dir] = lat; 2343 if (m->lat_min[dir] > lat || !m->lat_min[dir]) 2344 m->lat_min[dir] = lat; 2345 2346 m->nr_queued[dir]--; 2347 /* Push forward the busy start of monitor */ 2348 m->busy_start_ts[dir] = now; 2349 } 2350 spin_unlock_irqrestore(hba->host->host_lock, flags); 2351} 2352 2353/* Returns %true for SCSI commands and %false for device management commands. */ 2354static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd) 2355{ 2356 return !blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd)); 2357} 2358 2359/** 2360 * ufshcd_send_command - Send SCSI or device management commands 2361 * @hba: per adapter instance 2362 * @cmd: SCSI command or device management command pointer 2363 * @hwq: pointer to hardware queue instance 2364 */ 2365static inline void ufshcd_send_command(struct ufs_hba *hba, 2366 struct scsi_cmnd *cmd, 2367 struct ufs_hw_queue *hwq) 2368{ 2369 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2370 const int tag = scsi_cmd_to_rq(cmd)->tag; 2371 unsigned long flags; 2372 2373 if (hba->monitor.enabled) { 2374 lrbp->issue_time_stamp = ktime_get(); 2375 lrbp->issue_time_stamp_local_clock = local_clock(); 2376 lrbp->compl_time_stamp = ktime_set(0, 0); 2377 lrbp->compl_time_stamp_local_clock = 0; 2378 } 2379 if (ufshcd_is_scsi_cmd(cmd)) { 2380 ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND); 2381 ufshcd_clk_scaling_start_busy(hba); 2382 if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) 2383 ufshcd_start_monitor(hba, cmd); 2384 } 2385 2386 if (hba->mcq_enabled) { 2387 int utrd_size = sizeof(struct utp_transfer_req_desc); 2388 struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr; 2389 struct utp_transfer_req_desc *dest; 2390 2391 spin_lock(&hwq->sq_lock); 2392 dest = hwq->sqe_base_addr + hwq->sq_tail_slot; 2393 memcpy(dest, src, utrd_size); 2394 ufshcd_inc_sq_tail(hwq); 2395 spin_unlock(&hwq->sq_lock); 2396 } else { 2397 spin_lock_irqsave(&hba->outstanding_lock, flags); 2398 if (hba->vops && hba->vops->setup_xfer_req) 2399 hba->vops->setup_xfer_req(hba, tag, 2400 ufshcd_is_scsi_cmd(cmd)); 2401 __set_bit(tag, &hba->outstanding_reqs); 2402 ufshcd_writel(hba, 1 << tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); 2403 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 2404 } 2405} 2406 2407/** 2408 * ufshcd_copy_sense_data - Copy sense data in case of check condition 2409 * @cmd: SCSI command 2410 */ 2411static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd) 2412{ 2413 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2414 u8 *const sense_buffer = cmd->sense_buffer; 2415 u16 resp_len; 2416 int len; 2417 2418 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length); 2419 if (sense_buffer && resp_len) { 2420 int len_to_copy; 2421 2422 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); 2423 len_to_copy = min_t(int, UFS_SENSE_SIZE, len); 2424 2425 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data, 2426 len_to_copy); 2427 } 2428} 2429 2430/** 2431 * ufshcd_copy_query_response() - Copy the Query Response and the data 2432 * descriptor 2433 * @hba: per adapter instance 2434 * @lrbp: pointer to local reference block 2435 * 2436 * Return: 0 upon success; < 0 upon failure. 2437 */ 2438static 2439int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2440{ 2441 struct ufs_query_res *query_res = &hba->dev_cmd.query.response; 2442 2443 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE); 2444 2445 /* Get the descriptor */ 2446 if (hba->dev_cmd.query.descriptor && 2447 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) { 2448 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + 2449 GENERAL_UPIU_REQUEST_SIZE; 2450 u16 resp_len; 2451 u16 buf_len; 2452 2453 /* data segment length */ 2454 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header 2455 .data_segment_length); 2456 buf_len = be16_to_cpu( 2457 hba->dev_cmd.query.request.upiu_req.length); 2458 if (likely(buf_len >= resp_len)) { 2459 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len); 2460 } else { 2461 dev_warn(hba->dev, 2462 "%s: rsp size %d is bigger than buffer size %d", 2463 __func__, resp_len, buf_len); 2464 return -EINVAL; 2465 } 2466 } 2467 2468 return 0; 2469} 2470 2471/** 2472 * ufshcd_hba_capabilities - Read controller capabilities 2473 * @hba: per adapter instance 2474 * 2475 * Return: 0 on success, negative on error. 2476 */ 2477static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) 2478{ 2479 int err; 2480 2481 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); 2482 2483 /* nutrs and nutmrs are 0 based values */ 2484 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1; 2485 hba->nutmrs = 2486 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; 2487 2488 hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1; 2489 2490 /* Read crypto capabilities */ 2491 err = ufshcd_hba_init_crypto_capabilities(hba); 2492 if (err) { 2493 dev_err(hba->dev, "crypto setup failed\n"); 2494 return err; 2495 } 2496 2497 /* 2498 * The UFSHCI 3.0 specification does not define MCQ_SUPPORT and 2499 * LSDB_SUPPORT, but [31:29] as reserved bits with reset value 0s, which 2500 * means we can simply read values regardless of version. 2501 */ 2502 hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities); 2503 /* 2504 * 0h: legacy single doorbell support is available 2505 * 1h: indicate that legacy single doorbell support has been removed 2506 */ 2507 if (!(hba->quirks & UFSHCD_QUIRK_BROKEN_LSDBS_CAP)) 2508 hba->lsdb_sup = !FIELD_GET(MASK_LSDB_SUPPORT, hba->capabilities); 2509 else 2510 hba->lsdb_sup = true; 2511 2512 hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP); 2513 2514 return 0; 2515} 2516 2517/** 2518 * ufshcd_ready_for_uic_cmd - Check if controller is ready 2519 * to accept UIC commands 2520 * @hba: per adapter instance 2521 * 2522 * Return: true on success, else false. 2523 */ 2524static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) 2525{ 2526 u32 val; 2527 int ret = read_poll_timeout(ufshcd_readl, val, val & UIC_COMMAND_READY, 2528 500, uic_cmd_timeout * 1000, false, hba, 2529 REG_CONTROLLER_STATUS); 2530 return ret == 0; 2531} 2532 2533/** 2534 * ufshcd_get_upmcrs - Get the power mode change request status 2535 * @hba: Pointer to adapter instance 2536 * 2537 * This function gets the UPMCRS field of HCS register 2538 * 2539 * Return: value of UPMCRS field. 2540 */ 2541static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) 2542{ 2543 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7; 2544} 2545 2546/** 2547 * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer 2548 * @hba: per adapter instance 2549 * @uic_cmd: UIC command 2550 */ 2551static inline void 2552ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2553{ 2554 lockdep_assert_held(&hba->uic_cmd_mutex); 2555 2556 WARN_ON(hba->active_uic_cmd); 2557 2558 hba->active_uic_cmd = uic_cmd; 2559 2560 /* Write Args */ 2561 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1); 2562 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2); 2563 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3); 2564 2565 ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND); 2566 2567 /* Write UIC Cmd */ 2568 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK, 2569 REG_UIC_COMMAND); 2570} 2571 2572/** 2573 * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command 2574 * @hba: per adapter instance 2575 * @uic_cmd: UIC command 2576 * 2577 * Return: 0 only if success. 2578 */ 2579static int 2580ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2581{ 2582 int ret; 2583 unsigned long flags; 2584 2585 lockdep_assert_held(&hba->uic_cmd_mutex); 2586 2587 if (wait_for_completion_timeout(&uic_cmd->done, 2588 msecs_to_jiffies(uic_cmd_timeout))) { 2589 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 2590 } else { 2591 ret = -ETIMEDOUT; 2592 dev_err(hba->dev, 2593 "uic cmd 0x%x with arg3 0x%x completion timeout\n", 2594 uic_cmd->command, uic_cmd->argument3); 2595 2596 if (!uic_cmd->cmd_active) { 2597 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n", 2598 __func__); 2599 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 2600 } 2601 } 2602 2603 spin_lock_irqsave(hba->host->host_lock, flags); 2604 hba->active_uic_cmd = NULL; 2605 spin_unlock_irqrestore(hba->host->host_lock, flags); 2606 2607 return ret; 2608} 2609 2610/** 2611 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 2612 * @hba: per adapter instance 2613 * @uic_cmd: UIC command 2614 * 2615 * Return: 0 if successful; < 0 upon failure. 2616 */ 2617static int 2618__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2619{ 2620 lockdep_assert_held(&hba->uic_cmd_mutex); 2621 2622 if (!ufshcd_ready_for_uic_cmd(hba)) { 2623 dev_err(hba->dev, 2624 "Controller not ready to accept UIC commands\n"); 2625 return -EIO; 2626 } 2627 2628 init_completion(&uic_cmd->done); 2629 2630 uic_cmd->cmd_active = true; 2631 ufshcd_dispatch_uic_cmd(hba, uic_cmd); 2632 2633 return 0; 2634} 2635 2636/** 2637 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 2638 * @hba: per adapter instance 2639 * @uic_cmd: UIC command 2640 * 2641 * Return: 0 only if success. 2642 */ 2643int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2644{ 2645 unsigned long flags; 2646 int ret; 2647 2648 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) 2649 return 0; 2650 2651 ufshcd_hold(hba); 2652 mutex_lock(&hba->uic_cmd_mutex); 2653 ufshcd_add_delay_before_dme_cmd(hba); 2654 2655 spin_lock_irqsave(hba->host->host_lock, flags); 2656 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); 2657 spin_unlock_irqrestore(hba->host->host_lock, flags); 2658 2659 ret = __ufshcd_send_uic_cmd(hba, uic_cmd); 2660 if (!ret) 2661 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd); 2662 2663 mutex_unlock(&hba->uic_cmd_mutex); 2664 2665 ufshcd_release(hba); 2666 return ret; 2667} 2668 2669/** 2670 * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format) 2671 * @hba: per-adapter instance 2672 * @lrbp: pointer to local reference block 2673 * @sg_entries: The number of sg lists actually used 2674 * @sg_list: Pointer to SG list 2675 */ 2676static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int sg_entries, 2677 struct scatterlist *sg_list) 2678{ 2679 struct ufshcd_sg_entry *prd; 2680 struct scatterlist *sg; 2681 int i; 2682 2683 if (sg_entries) { 2684 2685 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) 2686 lrbp->utr_descriptor_ptr->prd_table_length = 2687 cpu_to_le16(sg_entries * ufshcd_sg_entry_size(hba)); 2688 else 2689 lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries); 2690 2691 prd = lrbp->ucd_prdt_ptr; 2692 2693 for_each_sg(sg_list, sg, sg_entries, i) { 2694 const unsigned int len = sg_dma_len(sg); 2695 2696 /* 2697 * From the UFSHCI spec: "Data Byte Count (DBC): A '0' 2698 * based value that indicates the length, in bytes, of 2699 * the data block. A maximum of length of 256KB may 2700 * exist for any entry. Bits 1:0 of this field shall be 2701 * 11b to indicate Dword granularity. A value of '3' 2702 * indicates 4 bytes, '7' indicates 8 bytes, etc." 2703 */ 2704 WARN_ONCE(len > SZ_256K, "len = %#x\n", len); 2705 prd->size = cpu_to_le32(len - 1); 2706 prd->addr = cpu_to_le64(sg->dma_address); 2707 prd->reserved = 0; 2708 prd = (void *)prd + ufshcd_sg_entry_size(hba); 2709 } 2710 } else { 2711 lrbp->utr_descriptor_ptr->prd_table_length = 0; 2712 } 2713} 2714 2715/** 2716 * ufshcd_map_sg - Map scatter-gather list to prdt 2717 * @hba: per adapter instance 2718 * @cmd: SCSI command 2719 * 2720 * Return: 0 in case of success, non-zero value in case of failure. 2721 */ 2722static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2723{ 2724 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2725 int sg_segments = scsi_dma_map(cmd); 2726 2727 if (sg_segments < 0) 2728 return sg_segments; 2729 2730 ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); 2731 2732 return ufshcd_crypto_fill_prdt(hba, cmd); 2733} 2734 2735/** 2736 * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request 2737 * descriptor according to request 2738 * @hba: per adapter instance 2739 * @lrbp: pointer to local reference block 2740 * @upiu_flags: flags required in the header 2741 * @cmd_dir: requests data direction 2742 * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments) 2743 */ 2744static void 2745ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 2746 u8 *upiu_flags, enum dma_data_direction cmd_dir, 2747 int ehs_length) 2748{ 2749 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; 2750 struct request_desc_header *h = &req_desc->header; 2751 enum utp_data_direction data_direction; 2752 2753 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; 2754 2755 *h = (typeof(*h)){ }; 2756 2757 if (cmd_dir == DMA_FROM_DEVICE) { 2758 data_direction = UTP_DEVICE_TO_HOST; 2759 *upiu_flags = UPIU_CMD_FLAGS_READ; 2760 } else if (cmd_dir == DMA_TO_DEVICE) { 2761 data_direction = UTP_HOST_TO_DEVICE; 2762 *upiu_flags = UPIU_CMD_FLAGS_WRITE; 2763 } else { 2764 data_direction = UTP_NO_DATA_TRANSFER; 2765 *upiu_flags = UPIU_CMD_FLAGS_NONE; 2766 } 2767 2768 h->command_type = lrbp->command_type; 2769 h->data_direction = data_direction; 2770 h->ehs_length = ehs_length; 2771 2772 if (lrbp->intr_cmd) 2773 h->interrupt = 1; 2774 2775 /* Prepare crypto related dwords */ 2776 ufshcd_prepare_req_desc_hdr_crypto(lrbp, h); 2777 2778 /* 2779 * assigning invalid value for command status. Controller 2780 * updates OCS on command completion, with the command 2781 * status 2782 */ 2783 h->ocs = OCS_INVALID_COMMAND_STATUS; 2784 2785 req_desc->prd_table_length = 0; 2786} 2787 2788/** 2789 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, 2790 * for scsi commands 2791 * @cmd: SCSI command 2792 * @upiu_flags: flags 2793 */ 2794static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, 2795 u8 upiu_flags) 2796{ 2797 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2798 const int tag = scsi_cmd_to_rq(cmd)->tag; 2799 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2800 unsigned short cdb_len; 2801 2802 ucd_req_ptr->header = (struct utp_upiu_header){ 2803 .transaction_code = UPIU_TRANSACTION_COMMAND, 2804 .flags = upiu_flags, 2805 .lun = lrbp->lun, 2806 .task_tag = tag, 2807 .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI, 2808 }; 2809 2810 WARN_ON_ONCE(ucd_req_ptr->header.task_tag != tag); 2811 2812 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length); 2813 2814 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE); 2815 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len); 2816 2817 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2818} 2819 2820/** 2821 * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request 2822 * @hba: UFS hba 2823 * @cmd: SCSI command pointer 2824 * @upiu_flags: flags 2825 */ 2826static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, 2827 struct scsi_cmnd *cmd, u8 upiu_flags) 2828{ 2829 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2830 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2831 const int tag = scsi_cmd_to_rq(cmd)->tag; 2832 struct ufs_query *query = &hba->dev_cmd.query; 2833 u16 len = be16_to_cpu(query->request.upiu_req.length); 2834 2835 /* Query request header */ 2836 ucd_req_ptr->header = (struct utp_upiu_header){ 2837 .transaction_code = UPIU_TRANSACTION_QUERY_REQ, 2838 .flags = upiu_flags, 2839 .lun = lrbp->lun, 2840 .task_tag = tag, 2841 .query_function = query->request.query_func, 2842 /* Data segment length only need for WRITE_DESC */ 2843 .data_segment_length = 2844 query->request.upiu_req.opcode == 2845 UPIU_QUERY_OPCODE_WRITE_DESC ? 2846 cpu_to_be16(len) : 2847 0, 2848 }; 2849 2850 /* Copy the Query Request buffer as is */ 2851 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, 2852 QUERY_OSF_SIZE); 2853 2854 /* Copy the Descriptor */ 2855 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) 2856 memcpy(ucd_req_ptr + 1, query->descriptor, len); 2857} 2858 2859static inline void ufshcd_prepare_utp_nop_upiu(struct scsi_cmnd *cmd) 2860{ 2861 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2862 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2863 const int tag = scsi_cmd_to_rq(cmd)->tag; 2864 2865 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); 2866 2867 ucd_req_ptr->header = (struct utp_upiu_header){ 2868 .transaction_code = UPIU_TRANSACTION_NOP_OUT, 2869 .task_tag = tag, 2870 }; 2871} 2872 2873/** 2874 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU) 2875 * for Device Management Purposes 2876 * @hba: per adapter instance 2877 * @cmd: SCSI command pointer 2878 * 2879 * Return: 0 upon success; < 0 upon failure. 2880 */ 2881static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, 2882 struct scsi_cmnd *cmd) 2883{ 2884 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2885 u8 upiu_flags; 2886 int ret = 0; 2887 2888 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 2889 2890 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) 2891 ufshcd_prepare_utp_query_req_upiu(hba, cmd, upiu_flags); 2892 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) 2893 ufshcd_prepare_utp_nop_upiu(cmd); 2894 else 2895 ret = -EINVAL; 2896 2897 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2898 2899 return ret; 2900} 2901 2902/** 2903 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) 2904 * for SCSI Purposes 2905 * @hba: per adapter instance 2906 * @cmd: SCSI command 2907 */ 2908static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2909{ 2910 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2911 struct request *rq = scsi_cmd_to_rq(cmd); 2912 unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); 2913 u8 upiu_flags; 2914 2915 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, 2916 cmd->sc_data_direction, 0); 2917 if (ioprio_class == IOPRIO_CLASS_RT) 2918 upiu_flags |= UPIU_CMD_FLAGS_CP; 2919 ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags); 2920} 2921 2922static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd) 2923{ 2924 const int i = scsi_cmd_to_rq(cmd)->tag; 2925 struct utp_transfer_cmd_desc *cmd_descp = 2926 (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba); 2927 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; 2928 dma_addr_t cmd_desc_element_addr = 2929 hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba); 2930 u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); 2931 u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); 2932 struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd); 2933 2934 lrb->utr_descriptor_ptr = utrdlp + i; 2935 lrb->utrd_dma_addr = 2936 hba->utrdl_dma_addr + i * sizeof(struct utp_transfer_req_desc); 2937 lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; 2938 lrb->ucd_req_dma_addr = cmd_desc_element_addr; 2939 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; 2940 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; 2941 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; 2942 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; 2943} 2944 2945static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 2946 u8 lun, int tag) 2947{ 2948 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2949 2950 ufshcd_init_lrb(hba, cmd); 2951 2952 memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); 2953 2954 lrbp->lun = lun; 2955 ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ? 2956 scsi_cmd_to_rq(cmd) : NULL, lrbp); 2957} 2958 2959static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 2960 u8 lun, int tag) 2961{ 2962 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 2963 2964 __ufshcd_setup_cmd(hba, cmd, lun, tag); 2965 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); 2966 lrbp->req_abort_skip = false; 2967 2968 ufshcd_comp_scsi_upiu(hba, cmd); 2969} 2970 2971/** 2972 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID 2973 * @upiu_wlun_id: UPIU W-LUN id 2974 * 2975 * Return: SCSI W-LUN id. 2976 */ 2977static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id) 2978{ 2979 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE; 2980} 2981 2982static inline bool is_device_wlun(struct scsi_device *sdev) 2983{ 2984 return sdev->lun == 2985 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN); 2986} 2987 2988/* 2989 * Associate the UFS controller queue with the default and poll HCTX types. 2990 * Initialize the mq_map[] arrays. 2991 */ 2992static void ufshcd_map_queues(struct Scsi_Host *shost) 2993{ 2994 struct ufs_hba *hba = shost_priv(shost); 2995 int i, queue_offset = 0; 2996 2997 if (!is_mcq_supported(hba)) { 2998 hba->nr_queues[HCTX_TYPE_DEFAULT] = 1; 2999 hba->nr_queues[HCTX_TYPE_READ] = 0; 3000 hba->nr_queues[HCTX_TYPE_POLL] = 1; 3001 hba->nr_hw_queues = 1; 3002 } 3003 3004 for (i = 0; i < shost->nr_maps; i++) { 3005 struct blk_mq_queue_map *map = &shost->tag_set.map[i]; 3006 3007 map->nr_queues = hba->nr_queues[i]; 3008 if (!map->nr_queues) 3009 continue; 3010 map->queue_offset = queue_offset; 3011 if (i == HCTX_TYPE_POLL && !is_mcq_supported(hba)) 3012 map->queue_offset = 0; 3013 3014 blk_mq_map_queues(map); 3015 queue_offset += map->nr_queues; 3016 } 3017} 3018 3019/* 3020 * The only purpose of this function is to make the SCSI core skip the memset() 3021 * call for the private command data. 3022 */ 3023static int ufshcd_init_cmd_priv(struct Scsi_Host *host, struct scsi_cmnd *cmd) 3024{ 3025 return 0; 3026} 3027 3028/** 3029 * ufshcd_queuecommand - main entry point for SCSI requests 3030 * @host: SCSI host pointer 3031 * @cmd: command from SCSI Midlayer 3032 * 3033 * Return: 0 for success, non-zero in case of failure. 3034 */ 3035static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) 3036{ 3037 struct ufs_hba *hba = shost_priv(host); 3038 int tag = scsi_cmd_to_rq(cmd)->tag; 3039 int err = 0; 3040 struct ufs_hw_queue *hwq = NULL; 3041 3042 switch (hba->ufshcd_state) { 3043 case UFSHCD_STATE_OPERATIONAL: 3044 break; 3045 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL: 3046 /* 3047 * SCSI error handler can call ->queuecommand() while UFS error 3048 * handler is in progress. Error interrupts could change the 3049 * state from UFSHCD_STATE_RESET to 3050 * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests 3051 * being issued in that case. 3052 */ 3053 if (ufshcd_eh_in_progress(hba)) { 3054 err = SCSI_MLQUEUE_HOST_BUSY; 3055 goto out; 3056 } 3057 break; 3058 case UFSHCD_STATE_EH_SCHEDULED_FATAL: 3059 /* 3060 * pm_runtime_get_sync() is used at error handling preparation 3061 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's 3062 * PM ops, it can never be finished if we let SCSI layer keep 3063 * retrying it, which gets err handler stuck forever. Neither 3064 * can we let the scsi cmd pass through, because UFS is in bad 3065 * state, the scsi cmd may eventually time out, which will get 3066 * err handler blocked for too long. So, just fail the scsi cmd 3067 * sent from PM ops, err handler can recover PM error anyways. 3068 */ 3069 if (hba->pm_op_in_progress) { 3070 hba->force_reset = true; 3071 set_host_byte(cmd, DID_BAD_TARGET); 3072 scsi_done(cmd); 3073 goto out; 3074 } 3075 fallthrough; 3076 case UFSHCD_STATE_RESET: 3077 err = SCSI_MLQUEUE_HOST_BUSY; 3078 goto out; 3079 case UFSHCD_STATE_ERROR: 3080 set_host_byte(cmd, DID_ERROR); 3081 scsi_done(cmd); 3082 goto out; 3083 } 3084 3085 hba->req_abort_count = 0; 3086 3087 ufshcd_hold(hba); 3088 3089 ufshcd_setup_scsi_cmd(hba, cmd, 3090 ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); 3091 3092 err = ufshcd_map_sg(hba, cmd); 3093 if (err) { 3094 ufshcd_release(hba); 3095 goto out; 3096 } 3097 3098 if (hba->mcq_enabled) 3099 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 3100 3101 ufshcd_send_command(hba, cmd, hwq); 3102 3103out: 3104 if (ufs_trigger_eh(hba)) { 3105 unsigned long flags; 3106 3107 spin_lock_irqsave(hba->host->host_lock, flags); 3108 ufshcd_schedule_eh_work(hba); 3109 spin_unlock_irqrestore(hba->host->host_lock, flags); 3110 } 3111 3112 return err; 3113} 3114 3115static int ufshcd_queue_reserved_command(struct Scsi_Host *host, 3116 struct scsi_cmnd *cmd) 3117{ 3118 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3119 struct request *rq = scsi_cmd_to_rq(cmd); 3120 struct ufs_hba *hba = shost_priv(host); 3121 struct ufs_hw_queue *hwq = 3122 hba->mcq_enabled ? ufshcd_mcq_req_to_hwq(hba, rq) : NULL; 3123 3124 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); 3125 ufshcd_send_command(hba, cmd, hwq); 3126 return 0; 3127} 3128 3129static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3130 enum dev_cmd_type cmd_type, u8 lun, int tag) 3131{ 3132 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3133 3134 __ufshcd_setup_cmd(hba, cmd, lun, tag); 3135 lrbp->intr_cmd = true; /* No interrupt aggregation */ 3136 hba->dev_cmd.type = cmd_type; 3137} 3138 3139/* 3140 * Return: 0 upon success; < 0 upon failure. 3141 */ 3142static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3143 enum dev_cmd_type cmd_type, int tag) 3144{ 3145 ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); 3146 3147 return ufshcd_compose_devman_upiu(hba, cmd); 3148} 3149 3150/* 3151 * Check with the block layer if the command is inflight 3152 * @cmd: command to check. 3153 * 3154 * Return: true if command is inflight; false if not. 3155 */ 3156bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd) 3157{ 3158 return cmd && blk_mq_rq_state(scsi_cmd_to_rq(cmd)) == MQ_RQ_IN_FLIGHT; 3159} 3160 3161/* 3162 * Clear the pending command in the controller and wait until 3163 * the controller confirms that the command has been cleared. 3164 * @hba: per adapter instance 3165 * @task_tag: The tag number of the command to be cleared. 3166 */ 3167static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag) 3168{ 3169 u32 mask; 3170 int err; 3171 3172 if (hba->mcq_enabled) { 3173 /* 3174 * MCQ mode. Clean up the MCQ resources similar to 3175 * what the ufshcd_utrl_clear() does for SDB mode. 3176 */ 3177 err = ufshcd_mcq_sq_cleanup(hba, task_tag); 3178 if (err) { 3179 dev_err(hba->dev, "%s: failed tag=%d. err=%d\n", 3180 __func__, task_tag, err); 3181 return err; 3182 } 3183 return 0; 3184 } 3185 3186 mask = 1U << task_tag; 3187 3188 /* clear outstanding transaction before retry */ 3189 ufshcd_utrl_clear(hba, mask); 3190 3191 /* 3192 * wait for h/w to clear corresponding bit in door-bell. 3193 * max. wait is 1 sec. 3194 */ 3195 return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL, 3196 mask, ~mask, 1000, 1000); 3197} 3198 3199/** 3200 * ufshcd_dev_cmd_completion() - handles device management command responses 3201 * @hba: per adapter instance 3202 * @lrbp: pointer to local reference block 3203 * 3204 * Return: 0 upon success; < 0 upon failure. 3205 */ 3206static int 3207ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 3208{ 3209 enum upiu_response_transaction resp; 3210 int err = 0; 3211 3212 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 3213 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); 3214 3215 switch (resp) { 3216 case UPIU_TRANSACTION_NOP_IN: 3217 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) { 3218 err = -EINVAL; 3219 dev_err(hba->dev, "%s: unexpected response %x\n", 3220 __func__, resp); 3221 } 3222 break; 3223 case UPIU_TRANSACTION_QUERY_RSP: { 3224 u8 response = lrbp->ucd_rsp_ptr->header.response; 3225 3226 if (response == 0) { 3227 err = ufshcd_copy_query_response(hba, lrbp); 3228 } else { 3229 err = -EINVAL; 3230 dev_err(hba->dev, "%s: unexpected response in Query RSP: %x\n", 3231 __func__, response); 3232 } 3233 break; 3234 } 3235 case UPIU_TRANSACTION_REJECT_UPIU: 3236 /* TODO: handle Reject UPIU Response */ 3237 err = -EPERM; 3238 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", 3239 __func__); 3240 break; 3241 case UPIU_TRANSACTION_RESPONSE: 3242 if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) { 3243 err = -EINVAL; 3244 dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp); 3245 } 3246 break; 3247 default: 3248 err = -EINVAL; 3249 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", 3250 __func__, resp); 3251 break; 3252 } 3253 3254 WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); 3255 return err; 3256} 3257 3258static void ufshcd_dev_man_lock(struct ufs_hba *hba) 3259{ 3260 ufshcd_hold(hba); 3261 mutex_lock(&hba->dev_cmd.lock); 3262 down_read(&hba->clk_scaling_lock); 3263} 3264 3265static void ufshcd_dev_man_unlock(struct ufs_hba *hba) 3266{ 3267 up_read(&hba->clk_scaling_lock); 3268 mutex_unlock(&hba->dev_cmd.lock); 3269 ufshcd_release(hba); 3270} 3271 3272static struct scsi_cmnd *ufshcd_get_dev_mgmt_cmd(struct ufs_hba *hba) 3273{ 3274 /* 3275 * The caller must hold this lock to guarantee that the NOWAIT 3276 * allocation will succeed. 3277 */ 3278 lockdep_assert_held(&hba->dev_cmd.lock); 3279 3280 return scsi_get_internal_cmd( 3281 hba->host->pseudo_sdev, DMA_TO_DEVICE, 3282 BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT); 3283} 3284 3285static void ufshcd_put_dev_mgmt_cmd(struct scsi_cmnd *cmd) 3286{ 3287 scsi_put_internal_cmd(cmd); 3288} 3289 3290/* 3291 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3292 * < 0 if another error occurred. 3293 */ 3294static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, 3295 const u32 tag, int timeout) 3296{ 3297 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3298 struct request *rq = scsi_cmd_to_rq(cmd); 3299 blk_status_t sts; 3300 3301 rq->timeout = timeout; 3302 sts = blk_execute_rq(rq, true); 3303 if (sts != BLK_STS_OK) 3304 return blk_status_to_errno(sts); 3305 return lrbp->utr_descriptor_ptr->header.ocs; 3306} 3307 3308/** 3309 * ufshcd_exec_dev_cmd - API for sending device management requests 3310 * @hba: UFS hba 3311 * @cmd_type: specifies the type (NOP, Query...) 3312 * @timeout: timeout in milliseconds 3313 * 3314 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3315 * < 0 if another error occurred. 3316 * 3317 * NOTE: Since there is only one available tag for device management commands, 3318 * it is expected you hold the hba->dev_cmd.lock mutex. 3319 */ 3320static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, 3321 enum dev_cmd_type cmd_type, int timeout) 3322{ 3323 struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); 3324 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 3325 u32 tag; 3326 int err; 3327 3328 /* Protects use of hba->dev_cmd. */ 3329 lockdep_assert_held(&hba->dev_cmd.lock); 3330 3331 if (WARN_ON_ONCE(!cmd)) 3332 return -ENOMEM; 3333 3334 tag = scsi_cmd_to_rq(cmd)->tag; 3335 3336 err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag); 3337 if (unlikely(err)) 3338 goto out; 3339 3340 err = ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); 3341 if (err == 0) 3342 err = ufshcd_dev_cmd_completion(hba, lrbp); 3343 3344out: 3345 ufshcd_put_dev_mgmt_cmd(cmd); 3346 3347 return err; 3348} 3349 3350/** 3351 * ufshcd_init_query() - init the query response and request parameters 3352 * @hba: per-adapter instance 3353 * @request: address of the request pointer to be initialized 3354 * @response: address of the response pointer to be initialized 3355 * @opcode: operation to perform 3356 * @idn: flag idn to access 3357 * @index: LU number to access 3358 * @selector: query/flag/descriptor further identification 3359 */ 3360static inline void ufshcd_init_query(struct ufs_hba *hba, 3361 struct ufs_query_req **request, struct ufs_query_res **response, 3362 enum query_opcode opcode, u8 idn, u8 index, u8 selector) 3363{ 3364 *request = &hba->dev_cmd.query.request; 3365 *response = &hba->dev_cmd.query.response; 3366 memset(*request, 0, sizeof(struct ufs_query_req)); 3367 memset(*response, 0, sizeof(struct ufs_query_res)); 3368 (*request)->upiu_req.opcode = opcode; 3369 (*request)->upiu_req.idn = idn; 3370 (*request)->upiu_req.index = index; 3371 (*request)->upiu_req.selector = selector; 3372} 3373 3374/* 3375 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3376 * < 0 if another error occurred. 3377 */ 3378static int ufshcd_query_flag_retry(struct ufs_hba *hba, 3379 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) 3380{ 3381 int ret; 3382 int retries; 3383 3384 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) { 3385 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res); 3386 if (ret) 3387 dev_dbg(hba->dev, 3388 "%s: failed with error %d, retries %d\n", 3389 __func__, ret, retries); 3390 else 3391 break; 3392 } 3393 3394 if (ret) 3395 dev_err(hba->dev, 3396 "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n", 3397 __func__, opcode, idn, ret, retries); 3398 return ret; 3399} 3400 3401/** 3402 * ufshcd_query_flag() - API function for sending flag query requests 3403 * @hba: per-adapter instance 3404 * @opcode: flag query to perform 3405 * @idn: flag idn to access 3406 * @index: flag index to access 3407 * @flag_res: the flag value after the query request completes 3408 * 3409 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3410 * < 0 if another error occurred. 3411 */ 3412int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, 3413 enum flag_idn idn, u8 index, bool *flag_res) 3414{ 3415 struct ufs_query_req *request = NULL; 3416 struct ufs_query_res *response = NULL; 3417 int err, selector = 0; 3418 int timeout = dev_cmd_timeout; 3419 3420 BUG_ON(!hba); 3421 3422 ufshcd_dev_man_lock(hba); 3423 3424 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3425 selector); 3426 3427 switch (opcode) { 3428 case UPIU_QUERY_OPCODE_SET_FLAG: 3429 case UPIU_QUERY_OPCODE_CLEAR_FLAG: 3430 case UPIU_QUERY_OPCODE_TOGGLE_FLAG: 3431 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3432 break; 3433 case UPIU_QUERY_OPCODE_READ_FLAG: 3434 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3435 if (!flag_res) { 3436 /* No dummy reads */ 3437 dev_err(hba->dev, "%s: Invalid argument for read request\n", 3438 __func__); 3439 err = -EINVAL; 3440 goto out_unlock; 3441 } 3442 break; 3443 default: 3444 dev_err(hba->dev, 3445 "%s: Expected query flag opcode but got = %d\n", 3446 __func__, opcode); 3447 err = -EINVAL; 3448 goto out_unlock; 3449 } 3450 3451 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout); 3452 3453 if (err) { 3454 dev_err(hba->dev, 3455 "%s: Sending flag query for idn %d failed, err = %d\n", 3456 __func__, idn, err); 3457 goto out_unlock; 3458 } 3459 3460 if (flag_res) 3461 *flag_res = (be32_to_cpu(response->upiu_res.value) & 3462 MASK_QUERY_UPIU_FLAG_LOC) & 0x1; 3463 3464out_unlock: 3465 ufshcd_dev_man_unlock(hba); 3466 return err; 3467} 3468 3469/** 3470 * ufshcd_query_attr - API function for sending attribute requests 3471 * @hba: per-adapter instance 3472 * @opcode: attribute opcode 3473 * @idn: attribute idn to access 3474 * @index: index field 3475 * @selector: selector field 3476 * @attr_val: the attribute value after the query request completes 3477 * 3478 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3479 * < 0 if another error occurred. 3480 */ 3481int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, 3482 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) 3483{ 3484 struct ufs_query_req *request = NULL; 3485 struct ufs_query_res *response = NULL; 3486 int err; 3487 3488 BUG_ON(!hba); 3489 3490 if (!attr_val) { 3491 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n", 3492 __func__, opcode); 3493 return -EINVAL; 3494 } 3495 3496 ufshcd_dev_man_lock(hba); 3497 3498 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3499 selector); 3500 3501 switch (opcode) { 3502 case UPIU_QUERY_OPCODE_WRITE_ATTR: 3503 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3504 request->upiu_req.value = cpu_to_be32(*attr_val); 3505 break; 3506 case UPIU_QUERY_OPCODE_READ_ATTR: 3507 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3508 break; 3509 default: 3510 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n", 3511 __func__, opcode); 3512 err = -EINVAL; 3513 goto out_unlock; 3514 } 3515 3516 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 3517 3518 if (err) { 3519 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", 3520 __func__, opcode, idn, index, err); 3521 goto out_unlock; 3522 } 3523 3524 *attr_val = be32_to_cpu(response->upiu_res.value); 3525 3526out_unlock: 3527 ufshcd_dev_man_unlock(hba); 3528 return err; 3529} 3530 3531/** 3532 * ufshcd_query_attr_retry() - API function for sending query 3533 * attribute with retries 3534 * @hba: per-adapter instance 3535 * @opcode: attribute opcode 3536 * @idn: attribute idn to access 3537 * @index: index field 3538 * @selector: selector field 3539 * @attr_val: the attribute value after the query request 3540 * completes 3541 * 3542 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3543 * < 0 if another error occurred. 3544 */ 3545int ufshcd_query_attr_retry(struct ufs_hba *hba, 3546 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, 3547 u32 *attr_val) 3548{ 3549 int ret = 0; 3550 u32 retries; 3551 3552 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 3553 ret = ufshcd_query_attr(hba, opcode, idn, index, 3554 selector, attr_val); 3555 if (ret) 3556 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n", 3557 __func__, ret, retries); 3558 else 3559 break; 3560 } 3561 3562 if (ret) 3563 dev_err(hba->dev, 3564 "%s: query attribute, idn %d, failed with error %d after %d retries\n", 3565 __func__, idn, ret, QUERY_REQ_RETRIES); 3566 return ret; 3567} 3568 3569/* 3570 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3571 * < 0 if another error occurred. 3572 */ 3573static int __ufshcd_query_descriptor(struct ufs_hba *hba, 3574 enum query_opcode opcode, enum desc_idn idn, u8 index, 3575 u8 selector, u8 *desc_buf, int *buf_len) 3576{ 3577 struct ufs_query_req *request = NULL; 3578 struct ufs_query_res *response = NULL; 3579 int err; 3580 3581 BUG_ON(!hba); 3582 3583 if (!desc_buf) { 3584 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n", 3585 __func__, opcode); 3586 return -EINVAL; 3587 } 3588 3589 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) { 3590 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n", 3591 __func__, *buf_len); 3592 return -EINVAL; 3593 } 3594 3595 ufshcd_dev_man_lock(hba); 3596 3597 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3598 selector); 3599 hba->dev_cmd.query.descriptor = desc_buf; 3600 request->upiu_req.length = cpu_to_be16(*buf_len); 3601 3602 switch (opcode) { 3603 case UPIU_QUERY_OPCODE_WRITE_DESC: 3604 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3605 break; 3606 case UPIU_QUERY_OPCODE_READ_DESC: 3607 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3608 break; 3609 default: 3610 dev_err(hba->dev, 3611 "%s: Expected query descriptor opcode but got = 0x%.2x\n", 3612 __func__, opcode); 3613 err = -EINVAL; 3614 goto out_unlock; 3615 } 3616 3617 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 3618 3619 if (err) { 3620 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", 3621 __func__, opcode, idn, index, err); 3622 goto out_unlock; 3623 } 3624 3625 *buf_len = be16_to_cpu(response->upiu_res.length); 3626 3627out_unlock: 3628 hba->dev_cmd.query.descriptor = NULL; 3629 ufshcd_dev_man_unlock(hba); 3630 return err; 3631} 3632 3633/** 3634 * ufshcd_query_descriptor_retry - API function for sending descriptor requests 3635 * @hba: per-adapter instance 3636 * @opcode: attribute opcode 3637 * @idn: attribute idn to access 3638 * @index: index field 3639 * @selector: selector field 3640 * @desc_buf: the buffer that contains the descriptor 3641 * @buf_len: length parameter passed to the device 3642 * 3643 * The buf_len parameter will contain, on return, the length parameter 3644 * received on the response. 3645 * 3646 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3647 * < 0 if another error occurred. 3648 */ 3649int ufshcd_query_descriptor_retry(struct ufs_hba *hba, 3650 enum query_opcode opcode, 3651 enum desc_idn idn, u8 index, 3652 u8 selector, 3653 u8 *desc_buf, int *buf_len) 3654{ 3655 int err; 3656 int retries; 3657 3658 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 3659 err = __ufshcd_query_descriptor(hba, opcode, idn, index, 3660 selector, desc_buf, buf_len); 3661 if (!err || err == -EINVAL) 3662 break; 3663 } 3664 3665 return err; 3666} 3667 3668/** 3669 * ufshcd_read_desc_param - read the specified descriptor parameter 3670 * @hba: Pointer to adapter instance 3671 * @desc_id: descriptor idn value 3672 * @desc_index: descriptor index 3673 * @param_offset: offset of the parameter to read 3674 * @param_read_buf: pointer to buffer where parameter would be read 3675 * @param_size: sizeof(param_read_buf) 3676 * 3677 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 3678 * < 0 if another error occurred. 3679 */ 3680int ufshcd_read_desc_param(struct ufs_hba *hba, 3681 enum desc_idn desc_id, 3682 int desc_index, 3683 u8 param_offset, 3684 u8 *param_read_buf, 3685 u8 param_size) 3686{ 3687 int ret; 3688 u8 *desc_buf; 3689 int buff_len = QUERY_DESC_MAX_SIZE; 3690 bool is_kmalloc = true; 3691 3692 /* Safety check */ 3693 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size) 3694 return -EINVAL; 3695 3696 /* Check whether we need temp memory */ 3697 if (param_offset != 0 || param_size < buff_len) { 3698 desc_buf = kzalloc(buff_len, GFP_KERNEL); 3699 if (!desc_buf) 3700 return -ENOMEM; 3701 } else { 3702 desc_buf = param_read_buf; 3703 is_kmalloc = false; 3704 } 3705 3706 /* Request for full descriptor */ 3707 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, 3708 desc_id, desc_index, 0, 3709 desc_buf, &buff_len); 3710 if (ret) { 3711 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n", 3712 __func__, desc_id, desc_index, param_offset, ret); 3713 goto out; 3714 } 3715 3716 /* Update descriptor length */ 3717 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET]; 3718 3719 if (param_offset >= buff_len) { 3720 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n", 3721 __func__, param_offset, desc_id, buff_len); 3722 ret = -EINVAL; 3723 goto out; 3724 } 3725 3726 /* Sanity check */ 3727 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) { 3728 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n", 3729 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]); 3730 ret = -EINVAL; 3731 goto out; 3732 } 3733 3734 if (is_kmalloc) { 3735 /* Make sure we don't copy more data than available */ 3736 if (param_offset >= buff_len) 3737 ret = -EINVAL; 3738 else 3739 memcpy(param_read_buf, &desc_buf[param_offset], 3740 min_t(u32, param_size, buff_len - param_offset)); 3741 } 3742out: 3743 if (is_kmalloc) 3744 kfree(desc_buf); 3745 return ret; 3746} 3747 3748/** 3749 * struct uc_string_id - unicode string 3750 * 3751 * @len: size of this descriptor inclusive 3752 * @type: descriptor type 3753 * @uc: unicode string character 3754 */ 3755struct uc_string_id { 3756 u8 len; 3757 u8 type; 3758 wchar_t uc[]; 3759} __packed; 3760 3761/* replace non-printable or non-ASCII characters with spaces */ 3762static inline char ufshcd_remove_non_printable(u8 ch) 3763{ 3764 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' '; 3765} 3766 3767/** 3768 * ufshcd_read_string_desc - read string descriptor 3769 * @hba: pointer to adapter instance 3770 * @desc_index: descriptor index 3771 * @buf: pointer to buffer where descriptor would be read, 3772 * the caller should free the memory. 3773 * @fmt: if %SD_ASCII_STD, convert from UTF-16 to ASCII 3774 * 3775 * Return: 3776 * * string size on success. 3777 * * -ENOMEM: on allocation failure 3778 * * -EINVAL: on a wrong parameter 3779 */ 3780int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum ufs_descr_fmt fmt) 3781{ 3782 struct uc_string_id *uc_str; 3783 u8 *str; 3784 int ret; 3785 3786 if (!buf) 3787 return -EINVAL; 3788 3789 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 3790 if (!uc_str) 3791 return -ENOMEM; 3792 3793 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0, 3794 (u8 *)uc_str, QUERY_DESC_MAX_SIZE); 3795 if (ret < 0) { 3796 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n", 3797 QUERY_REQ_RETRIES, ret); 3798 str = NULL; 3799 goto out; 3800 } 3801 3802 if (uc_str->len <= QUERY_DESC_HDR_SIZE) { 3803 dev_dbg(hba->dev, "String Desc is of zero length\n"); 3804 str = NULL; 3805 ret = 0; 3806 goto out; 3807 } 3808 3809 if (fmt == SD_ASCII_STD) { 3810 ssize_t ascii_len; 3811 int i; 3812 /* remove header and divide by 2 to move from UTF16 to UTF8 */ 3813 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1; 3814 str = kzalloc(ascii_len, GFP_KERNEL); 3815 if (!str) { 3816 ret = -ENOMEM; 3817 goto out; 3818 } 3819 3820 /* 3821 * the descriptor contains string in UTF16 format 3822 * we need to convert to utf-8 so it can be displayed 3823 */ 3824 ret = utf16s_to_utf8s(uc_str->uc, 3825 uc_str->len - QUERY_DESC_HDR_SIZE, 3826 UTF16_BIG_ENDIAN, str, ascii_len - 1); 3827 3828 /* replace non-printable or non-ASCII characters with spaces */ 3829 for (i = 0; i < ret; i++) 3830 str[i] = ufshcd_remove_non_printable(str[i]); 3831 3832 str[ret++] = '\0'; 3833 3834 } else { 3835 str = kmemdup(uc_str->uc, uc_str->len, GFP_KERNEL); 3836 if (!str) { 3837 ret = -ENOMEM; 3838 goto out; 3839 } 3840 ret = uc_str->len; 3841 } 3842out: 3843 *buf = str; 3844 kfree(uc_str); 3845 return ret; 3846} 3847 3848/** 3849 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter 3850 * @hba: Pointer to adapter instance 3851 * @lun: lun id 3852 * @param_offset: offset of the parameter to read 3853 * @param_read_buf: pointer to buffer where parameter would be read 3854 * @param_size: sizeof(param_read_buf) 3855 * 3856 * Return: 0 in case of success; < 0 upon failure. 3857 */ 3858static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba, 3859 int lun, 3860 enum unit_desc_param param_offset, 3861 u8 *param_read_buf, 3862 u32 param_size) 3863{ 3864 /* 3865 * Unit descriptors are only available for general purpose LUs (LUN id 3866 * from 0 to 7) and RPMB Well known LU. 3867 */ 3868 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun)) 3869 return -EOPNOTSUPP; 3870 3871 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun, 3872 param_offset, param_read_buf, param_size); 3873} 3874 3875static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba) 3876{ 3877 int err = 0; 3878 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US; 3879 3880 if (hba->dev_info.wspecversion >= 0x300) { 3881 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 3882 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0, 3883 &gating_wait); 3884 if (err) 3885 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n", 3886 err, gating_wait); 3887 3888 if (gating_wait == 0) { 3889 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US; 3890 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n", 3891 gating_wait); 3892 } 3893 3894 hba->dev_info.clk_gating_wait_us = gating_wait; 3895 } 3896 3897 return err; 3898} 3899 3900/** 3901 * ufshcd_memory_alloc - allocate memory for host memory space data structures 3902 * @hba: per adapter instance 3903 * 3904 * 1. Allocate DMA memory for Command Descriptor array 3905 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT 3906 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL). 3907 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List 3908 * (UTMRDL) 3909 * 4. Allocate memory for local reference block(lrb). 3910 * 3911 * Return: 0 for success, non-zero in case of failure. 3912 */ 3913static int ufshcd_memory_alloc(struct ufs_hba *hba) 3914{ 3915 size_t utmrdl_size, utrdl_size, ucdl_size; 3916 3917 /* Allocate memory for UTP command descriptors */ 3918 ucdl_size = ufshcd_get_ucd_size(hba) * hba->nutrs; 3919 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev, 3920 ucdl_size, 3921 &hba->ucdl_dma_addr, 3922 GFP_KERNEL); 3923 3924 /* 3925 * UFSHCI requires UTP command descriptor to be 128 byte aligned. 3926 */ 3927 if (!hba->ucdl_base_addr || 3928 WARN_ON(hba->ucdl_dma_addr & (128 - 1))) { 3929 dev_err(hba->dev, 3930 "Command Descriptor Memory allocation failed\n"); 3931 goto out; 3932 } 3933 3934 /* 3935 * Allocate memory for UTP Transfer descriptors 3936 * UFSHCI requires 1KB alignment of UTRD 3937 */ 3938 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs); 3939 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev, 3940 utrdl_size, 3941 &hba->utrdl_dma_addr, 3942 GFP_KERNEL); 3943 if (!hba->utrdl_base_addr || 3944 WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) { 3945 dev_err(hba->dev, 3946 "Transfer Descriptor Memory allocation failed\n"); 3947 goto out; 3948 } 3949 3950 /* 3951 * Skip utmrdl allocation; it may have been 3952 * allocated during first pass and not released during 3953 * MCQ memory allocation. 3954 * See ufshcd_release_sdb_queue() and ufshcd_config_mcq() 3955 */ 3956 if (hba->utmrdl_base_addr) 3957 goto skip_utmrdl; 3958 /* 3959 * Allocate memory for UTP Task Management descriptors 3960 * UFSHCI requires 1KB alignment of UTMRD 3961 */ 3962 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs; 3963 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev, 3964 utmrdl_size, 3965 &hba->utmrdl_dma_addr, 3966 GFP_KERNEL); 3967 if (!hba->utmrdl_base_addr || 3968 WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) { 3969 dev_err(hba->dev, 3970 "Task Management Descriptor Memory allocation failed\n"); 3971 goto out; 3972 } 3973 3974skip_utmrdl: 3975 return 0; 3976out: 3977 return -ENOMEM; 3978} 3979 3980/** 3981 * ufshcd_host_memory_configure - configure local reference block with 3982 * memory offsets 3983 * @hba: per adapter instance 3984 * 3985 * Configure Host memory space 3986 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA 3987 * address. 3988 * 2. Update each UTRD with Response UPIU offset, Response UPIU length 3989 * and PRDT offset. 3990 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT 3991 * into local reference block. 3992 */ 3993static void ufshcd_host_memory_configure(struct ufs_hba *hba) 3994{ 3995 struct utp_transfer_req_desc *utrdlp; 3996 dma_addr_t cmd_desc_dma_addr; 3997 dma_addr_t cmd_desc_element_addr; 3998 u16 response_offset; 3999 u16 prdt_offset; 4000 int cmd_desc_size; 4001 int i; 4002 4003 utrdlp = hba->utrdl_base_addr; 4004 4005 response_offset = 4006 offsetof(struct utp_transfer_cmd_desc, response_upiu); 4007 prdt_offset = 4008 offsetof(struct utp_transfer_cmd_desc, prd_table); 4009 4010 cmd_desc_size = ufshcd_get_ucd_size(hba); 4011 cmd_desc_dma_addr = hba->ucdl_dma_addr; 4012 4013 for (i = 0; i < hba->nutrs; i++) { 4014 /* Configure UTRD with command descriptor base address */ 4015 cmd_desc_element_addr = 4016 (cmd_desc_dma_addr + (cmd_desc_size * i)); 4017 utrdlp[i].command_desc_base_addr = 4018 cpu_to_le64(cmd_desc_element_addr); 4019 4020 /* Response upiu and prdt offset should be in double words */ 4021 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) { 4022 utrdlp[i].response_upiu_offset = 4023 cpu_to_le16(response_offset); 4024 utrdlp[i].prd_table_offset = 4025 cpu_to_le16(prdt_offset); 4026 utrdlp[i].response_upiu_length = 4027 cpu_to_le16(ALIGNED_UPIU_SIZE); 4028 } else { 4029 utrdlp[i].response_upiu_offset = 4030 cpu_to_le16(response_offset >> 2); 4031 utrdlp[i].prd_table_offset = 4032 cpu_to_le16(prdt_offset >> 2); 4033 utrdlp[i].response_upiu_length = 4034 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); 4035 } 4036 } 4037} 4038 4039/** 4040 * ufshcd_dme_link_startup - Notify Unipro to perform link startup 4041 * @hba: per adapter instance 4042 * 4043 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer, 4044 * in order to initialize the Unipro link startup procedure. 4045 * Once the Unipro links are up, the device connected to the controller 4046 * is detected. 4047 * 4048 * Return: 0 on success, non-zero value on failure. 4049 */ 4050static int ufshcd_dme_link_startup(struct ufs_hba *hba) 4051{ 4052 struct uic_command uic_cmd = { 4053 .command = UIC_CMD_DME_LINK_STARTUP, 4054 }; 4055 int ret; 4056 4057 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4058 if (ret) 4059 dev_dbg(hba->dev, 4060 "dme-link-startup: error code %d\n", ret); 4061 return ret; 4062} 4063/** 4064 * ufshcd_dme_reset - UIC command for DME_RESET 4065 * @hba: per adapter instance 4066 * 4067 * DME_RESET command is issued in order to reset UniPro stack. 4068 * This function now deals with cold reset. 4069 * 4070 * Return: 0 on success, non-zero value on failure. 4071 */ 4072int ufshcd_dme_reset(struct ufs_hba *hba) 4073{ 4074 struct uic_command uic_cmd = { 4075 .command = UIC_CMD_DME_RESET, 4076 }; 4077 int ret; 4078 4079 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4080 if (ret) 4081 dev_err(hba->dev, 4082 "dme-reset: error code %d\n", ret); 4083 4084 return ret; 4085} 4086EXPORT_SYMBOL_GPL(ufshcd_dme_reset); 4087 4088int ufshcd_dme_configure_adapt(struct ufs_hba *hba, 4089 int agreed_gear, 4090 int adapt_val) 4091{ 4092 int ret; 4093 4094 if (agreed_gear < UFS_HS_G4) 4095 adapt_val = PA_NO_ADAPT; 4096 4097 ret = ufshcd_dme_set(hba, 4098 UIC_ARG_MIB(PA_TXHSADAPTTYPE), 4099 adapt_val); 4100 return ret; 4101} 4102EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt); 4103 4104/** 4105 * ufshcd_dme_enable - UIC command for DME_ENABLE 4106 * @hba: per adapter instance 4107 * 4108 * DME_ENABLE command is issued in order to enable UniPro stack. 4109 * 4110 * Return: 0 on success, non-zero value on failure. 4111 */ 4112int ufshcd_dme_enable(struct ufs_hba *hba) 4113{ 4114 struct uic_command uic_cmd = { 4115 .command = UIC_CMD_DME_ENABLE, 4116 }; 4117 int ret; 4118 4119 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4120 if (ret) 4121 dev_err(hba->dev, 4122 "dme-enable: error code %d\n", ret); 4123 4124 return ret; 4125} 4126EXPORT_SYMBOL_GPL(ufshcd_dme_enable); 4127 4128static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) 4129{ 4130 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000 4131 unsigned long min_sleep_time_us; 4132 4133 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS)) 4134 return; 4135 4136 /* 4137 * last_dme_cmd_tstamp will be 0 only for 1st call to 4138 * this function 4139 */ 4140 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) { 4141 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US; 4142 } else { 4143 unsigned long delta = 4144 (unsigned long) ktime_to_us( 4145 ktime_sub(ktime_get(), 4146 hba->last_dme_cmd_tstamp)); 4147 4148 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US) 4149 min_sleep_time_us = 4150 MIN_DELAY_BEFORE_DME_CMDS_US - delta; 4151 else 4152 min_sleep_time_us = 0; /* no more delay required */ 4153 } 4154 4155 if (min_sleep_time_us > 0) { 4156 /* allow sleep for extra 50us if needed */ 4157 usleep_range(min_sleep_time_us, min_sleep_time_us + 50); 4158 } 4159 4160 /* update the last_dme_cmd_tstamp */ 4161 hba->last_dme_cmd_tstamp = ktime_get(); 4162} 4163 4164/** 4165 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET 4166 * @hba: per adapter instance 4167 * @attr_sel: uic command argument1 4168 * @attr_set: attribute set type as uic command argument2 4169 * @mib_val: setting value as uic command argument3 4170 * @peer: indicate whether peer or local 4171 * 4172 * Return: 0 on success, non-zero value on failure. 4173 */ 4174int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, 4175 u8 attr_set, u32 mib_val, u8 peer) 4176{ 4177 struct uic_command uic_cmd = { 4178 .command = peer ? UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET, 4179 .argument1 = attr_sel, 4180 .argument2 = UIC_ARG_ATTR_TYPE(attr_set), 4181 .argument3 = mib_val, 4182 }; 4183 static const char *const action[] = { 4184 "dme-set", 4185 "dme-peer-set" 4186 }; 4187 const char *set = action[!!peer]; 4188 int ret; 4189 int retries = UFS_UIC_COMMAND_RETRIES; 4190 4191 do { 4192 /* for peer attributes we retry upon failure */ 4193 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4194 if (ret) 4195 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n", 4196 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret); 4197 } while (ret && peer && --retries); 4198 4199 if (ret) 4200 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n", 4201 set, UIC_GET_ATTR_ID(attr_sel), mib_val, 4202 UFS_UIC_COMMAND_RETRIES - retries); 4203 4204 return ret; 4205} 4206EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); 4207 4208/** 4209 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET 4210 * @hba: per adapter instance 4211 * @attr_sel: uic command argument1 4212 * @mib_val: the value of the attribute as returned by the UIC command 4213 * @peer: indicate whether peer or local 4214 * 4215 * Return: 0 on success, non-zero value on failure. 4216 */ 4217int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, 4218 u32 *mib_val, u8 peer) 4219{ 4220 struct uic_command uic_cmd = { 4221 .command = peer ? UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET, 4222 .argument1 = attr_sel, 4223 }; 4224 static const char *const action[] = { 4225 "dme-get", 4226 "dme-peer-get" 4227 }; 4228 const char *get = action[!!peer]; 4229 int ret; 4230 int retries = UFS_UIC_COMMAND_RETRIES; 4231 struct ufs_pa_layer_attr orig_pwr_info; 4232 struct ufs_pa_layer_attr temp_pwr_info; 4233 bool pwr_mode_change = false; 4234 4235 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) { 4236 orig_pwr_info = hba->pwr_info; 4237 temp_pwr_info = orig_pwr_info; 4238 4239 if (orig_pwr_info.pwr_tx == FAST_MODE || 4240 orig_pwr_info.pwr_rx == FAST_MODE) { 4241 temp_pwr_info.pwr_tx = FASTAUTO_MODE; 4242 temp_pwr_info.pwr_rx = FASTAUTO_MODE; 4243 pwr_mode_change = true; 4244 } else if (orig_pwr_info.pwr_tx == SLOW_MODE || 4245 orig_pwr_info.pwr_rx == SLOW_MODE) { 4246 temp_pwr_info.pwr_tx = SLOWAUTO_MODE; 4247 temp_pwr_info.pwr_rx = SLOWAUTO_MODE; 4248 pwr_mode_change = true; 4249 } 4250 if (pwr_mode_change) { 4251 ret = ufshcd_change_power_mode(hba, &temp_pwr_info); 4252 if (ret) 4253 goto out; 4254 } 4255 } 4256 4257 do { 4258 /* for peer attributes we retry upon failure */ 4259 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4260 if (ret) 4261 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n", 4262 get, UIC_GET_ATTR_ID(attr_sel), ret); 4263 } while (ret && peer && --retries); 4264 4265 if (ret) 4266 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n", 4267 get, UIC_GET_ATTR_ID(attr_sel), 4268 UFS_UIC_COMMAND_RETRIES - retries); 4269 4270 if (mib_val) 4271 *mib_val = ret == 0 ? uic_cmd.argument3 : 0; 4272 4273 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE) 4274 && pwr_mode_change) 4275 ufshcd_change_power_mode(hba, &orig_pwr_info); 4276out: 4277 return ret; 4278} 4279EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); 4280 4281/** 4282 * ufshcd_dme_rmw - get modify set a DME attribute 4283 * @hba: per adapter instance 4284 * @mask: indicates which bits to clear from the value that has been read 4285 * @val: actual value to write 4286 * @attr: dme attribute 4287 */ 4288int ufshcd_dme_rmw(struct ufs_hba *hba, u32 mask, 4289 u32 val, u32 attr) 4290{ 4291 u32 cfg = 0; 4292 int err; 4293 4294 err = ufshcd_dme_get(hba, UIC_ARG_MIB(attr), &cfg); 4295 if (err) 4296 return err; 4297 4298 cfg &= ~mask; 4299 cfg |= (val & mask); 4300 4301 return ufshcd_dme_set(hba, UIC_ARG_MIB(attr), cfg); 4302} 4303EXPORT_SYMBOL_GPL(ufshcd_dme_rmw); 4304 4305/** 4306 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power 4307 * state) and waits for it to take effect. 4308 * 4309 * @hba: per adapter instance 4310 * @cmd: UIC command to execute 4311 * 4312 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER & 4313 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host 4314 * and device UniPro link and hence it's final completion would be indicated by 4315 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in 4316 * addition to normal UIC command completion Status (UCCS). This function only 4317 * returns after the relevant status bits indicate the completion. 4318 * 4319 * Return: 0 on success, non-zero value on failure. 4320 */ 4321static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) 4322{ 4323 DECLARE_COMPLETION_ONSTACK(uic_async_done); 4324 unsigned long flags; 4325 u8 status; 4326 int ret; 4327 4328 mutex_lock(&hba->uic_cmd_mutex); 4329 ufshcd_add_delay_before_dme_cmd(hba); 4330 4331 spin_lock_irqsave(hba->host->host_lock, flags); 4332 if (ufshcd_is_link_broken(hba)) { 4333 ret = -ENOLINK; 4334 goto out_unlock; 4335 } 4336 hba->uic_async_done = &uic_async_done; 4337 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL); 4338 spin_unlock_irqrestore(hba->host->host_lock, flags); 4339 ret = __ufshcd_send_uic_cmd(hba, cmd); 4340 if (ret) { 4341 dev_err(hba->dev, 4342 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", 4343 cmd->command, cmd->argument3, ret); 4344 goto out; 4345 } 4346 4347 if (!wait_for_completion_timeout(hba->uic_async_done, 4348 msecs_to_jiffies(uic_cmd_timeout))) { 4349 dev_err(hba->dev, 4350 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n", 4351 cmd->command, cmd->argument3); 4352 4353 if (!cmd->cmd_active) { 4354 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n", 4355 __func__); 4356 goto check_upmcrs; 4357 } 4358 4359 ret = -ETIMEDOUT; 4360 goto out; 4361 } 4362 4363check_upmcrs: 4364 status = ufshcd_get_upmcrs(hba); 4365 if (status != PWR_LOCAL) { 4366 dev_err(hba->dev, 4367 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n", 4368 cmd->command, status); 4369 ret = (status != PWR_OK) ? status : -1; 4370 } 4371out: 4372 if (ret) { 4373 ufshcd_print_host_state(hba); 4374 ufshcd_print_pwr_info(hba); 4375 ufshcd_print_evt_hist(hba); 4376 } 4377 4378 spin_lock_irqsave(hba->host->host_lock, flags); 4379 hba->active_uic_cmd = NULL; 4380 hba->uic_async_done = NULL; 4381 if (ret && !hba->pm_op_in_progress) { 4382 ufshcd_set_link_broken(hba); 4383 ufshcd_schedule_eh_work(hba); 4384 } 4385out_unlock: 4386 spin_unlock_irqrestore(hba->host->host_lock, flags); 4387 mutex_unlock(&hba->uic_cmd_mutex); 4388 4389 return ret; 4390} 4391 4392/** 4393 * ufshcd_send_bsg_uic_cmd - Send UIC commands requested via BSG layer and retrieve the result 4394 * @hba: per adapter instance 4395 * @uic_cmd: UIC command 4396 * 4397 * Return: 0 only if success. 4398 */ 4399int ufshcd_send_bsg_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 4400{ 4401 int ret; 4402 4403 if (uic_cmd->argument1 != UIC_ARG_MIB(PA_PWRMODE) || 4404 uic_cmd->command != UIC_CMD_DME_SET) 4405 return ufshcd_send_uic_cmd(hba, uic_cmd); 4406 4407 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) 4408 return 0; 4409 4410 ufshcd_hold(hba); 4411 ret = ufshcd_uic_pwr_ctrl(hba, uic_cmd); 4412 ufshcd_release(hba); 4413 4414 return ret; 4415} 4416 4417/** 4418 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage 4419 * using DME_SET primitives. 4420 * @hba: per adapter instance 4421 * @mode: powr mode value 4422 * 4423 * Return: 0 on success, non-zero value on failure. 4424 */ 4425int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) 4426{ 4427 struct uic_command uic_cmd = { 4428 .command = UIC_CMD_DME_SET, 4429 .argument1 = UIC_ARG_MIB(PA_PWRMODE), 4430 .argument3 = mode, 4431 }; 4432 int ret; 4433 4434 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) { 4435 ret = ufshcd_dme_set(hba, 4436 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1); 4437 if (ret) { 4438 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n", 4439 __func__, ret); 4440 goto out; 4441 } 4442 } 4443 4444 ufshcd_hold(hba); 4445 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4446 ufshcd_release(hba); 4447 4448out: 4449 return ret; 4450} 4451EXPORT_SYMBOL_GPL(ufshcd_uic_change_pwr_mode); 4452 4453int ufshcd_link_recovery(struct ufs_hba *hba) 4454{ 4455 int ret; 4456 unsigned long flags; 4457 4458 spin_lock_irqsave(hba->host->host_lock, flags); 4459 hba->ufshcd_state = UFSHCD_STATE_RESET; 4460 ufshcd_set_eh_in_progress(hba); 4461 spin_unlock_irqrestore(hba->host->host_lock, flags); 4462 4463 /* Reset the attached device */ 4464 ufshcd_device_reset(hba); 4465 4466 ret = ufshcd_host_reset_and_restore(hba); 4467 4468 spin_lock_irqsave(hba->host->host_lock, flags); 4469 if (ret) 4470 hba->ufshcd_state = UFSHCD_STATE_ERROR; 4471 ufshcd_clear_eh_in_progress(hba); 4472 spin_unlock_irqrestore(hba->host->host_lock, flags); 4473 4474 if (ret) 4475 dev_err(hba->dev, "%s: link recovery failed, err %d", 4476 __func__, ret); 4477 4478 return ret; 4479} 4480EXPORT_SYMBOL_GPL(ufshcd_link_recovery); 4481 4482int ufshcd_uic_hibern8_enter(struct ufs_hba *hba) 4483{ 4484 struct uic_command uic_cmd = { 4485 .command = UIC_CMD_DME_HIBER_ENTER, 4486 }; 4487 ktime_t start = ktime_get(); 4488 int ret; 4489 4490 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE); 4491 4492 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4493 trace_ufshcd_profile_hibern8(hba, "enter", 4494 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 4495 4496 if (ret) 4497 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n", 4498 __func__, ret); 4499 else 4500 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, 4501 POST_CHANGE); 4502 4503 return ret; 4504} 4505EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter); 4506 4507int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) 4508{ 4509 struct uic_command uic_cmd = { 4510 .command = UIC_CMD_DME_HIBER_EXIT, 4511 }; 4512 int ret; 4513 ktime_t start = ktime_get(); 4514 4515 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE); 4516 4517 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4518 trace_ufshcd_profile_hibern8(hba, "exit", 4519 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 4520 4521 if (ret) { 4522 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n", 4523 __func__, ret); 4524 } else { 4525 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, 4526 POST_CHANGE); 4527 hba->ufs_stats.last_hibern8_exit_tstamp = local_clock(); 4528 hba->ufs_stats.hibern8_exit_cnt++; 4529 } 4530 4531 return ret; 4532} 4533EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit); 4534 4535static void ufshcd_configure_auto_hibern8(struct ufs_hba *hba) 4536{ 4537 if (!ufshcd_is_auto_hibern8_supported(hba)) 4538 return; 4539 4540 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); 4541} 4542 4543void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit) 4544{ 4545 const u32 cur_ahit = READ_ONCE(hba->ahit); 4546 4547 if (!ufshcd_is_auto_hibern8_supported(hba) || cur_ahit == ahit) 4548 return; 4549 4550 WRITE_ONCE(hba->ahit, ahit); 4551 if (!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) { 4552 ufshcd_rpm_get_sync(hba); 4553 ufshcd_hold(hba); 4554 ufshcd_configure_auto_hibern8(hba); 4555 ufshcd_release(hba); 4556 ufshcd_rpm_put_sync(hba); 4557 } 4558} 4559EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update); 4560 4561 /** 4562 * ufshcd_init_pwr_info - setting the POR (power on reset) 4563 * values in hba power info 4564 * @hba: per-adapter instance 4565 */ 4566static void ufshcd_init_pwr_info(struct ufs_hba *hba) 4567{ 4568 hba->pwr_info.gear_rx = UFS_PWM_G1; 4569 hba->pwr_info.gear_tx = UFS_PWM_G1; 4570 hba->pwr_info.lane_rx = UFS_LANE_1; 4571 hba->pwr_info.lane_tx = UFS_LANE_1; 4572 hba->pwr_info.pwr_rx = SLOWAUTO_MODE; 4573 hba->pwr_info.pwr_tx = SLOWAUTO_MODE; 4574 hba->pwr_info.hs_rate = 0; 4575} 4576 4577/** 4578 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device 4579 * @hba: per-adapter instance 4580 * 4581 * Return: 0 upon success; < 0 upon failure. 4582 */ 4583static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) 4584{ 4585 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 4586 4587 if (hba->max_pwr_info.is_valid) 4588 return 0; 4589 4590 if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) { 4591 pwr_info->pwr_tx = FASTAUTO_MODE; 4592 pwr_info->pwr_rx = FASTAUTO_MODE; 4593 } else { 4594 pwr_info->pwr_tx = FAST_MODE; 4595 pwr_info->pwr_rx = FAST_MODE; 4596 } 4597 pwr_info->hs_rate = PA_HS_MODE_B; 4598 4599 /* Get the connected lane count */ 4600 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), 4601 &pwr_info->lane_rx); 4602 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4603 &pwr_info->lane_tx); 4604 4605 if (!pwr_info->lane_rx || !pwr_info->lane_tx) { 4606 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n", 4607 __func__, 4608 pwr_info->lane_rx, 4609 pwr_info->lane_tx); 4610 return -EINVAL; 4611 } 4612 4613 if (pwr_info->lane_rx != pwr_info->lane_tx) { 4614 dev_err(hba->dev, "%s: asymmetric connected lanes. rx=%d, tx=%d\n", 4615 __func__, 4616 pwr_info->lane_rx, 4617 pwr_info->lane_tx); 4618 return -EINVAL; 4619 } 4620 4621 /* 4622 * First, get the maximum gears of HS speed. 4623 * If a zero value, it means there is no HSGEAR capability. 4624 * Then, get the maximum gears of PWM speed. 4625 */ 4626 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx); 4627 if (!pwr_info->gear_rx) { 4628 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 4629 &pwr_info->gear_rx); 4630 if (!pwr_info->gear_rx) { 4631 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n", 4632 __func__, pwr_info->gear_rx); 4633 return -EINVAL; 4634 } 4635 pwr_info->pwr_rx = SLOW_MODE; 4636 } 4637 4638 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), 4639 &pwr_info->gear_tx); 4640 if (!pwr_info->gear_tx) { 4641 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 4642 &pwr_info->gear_tx); 4643 if (!pwr_info->gear_tx) { 4644 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n", 4645 __func__, pwr_info->gear_tx); 4646 return -EINVAL; 4647 } 4648 pwr_info->pwr_tx = SLOW_MODE; 4649 } 4650 4651 hba->max_pwr_info.is_valid = true; 4652 return 0; 4653} 4654 4655static int ufshcd_change_power_mode(struct ufs_hba *hba, 4656 struct ufs_pa_layer_attr *pwr_mode) 4657{ 4658 int ret; 4659 4660 /* if already configured to the requested pwr_mode */ 4661 if (!hba->force_pmc && 4662 pwr_mode->gear_rx == hba->pwr_info.gear_rx && 4663 pwr_mode->gear_tx == hba->pwr_info.gear_tx && 4664 pwr_mode->lane_rx == hba->pwr_info.lane_rx && 4665 pwr_mode->lane_tx == hba->pwr_info.lane_tx && 4666 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx && 4667 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx && 4668 pwr_mode->hs_rate == hba->pwr_info.hs_rate) { 4669 dev_dbg(hba->dev, "%s: power already configured\n", __func__); 4670 return 0; 4671 } 4672 4673 /* 4674 * Configure attributes for power mode change with below. 4675 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION, 4676 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION, 4677 * - PA_HSSERIES 4678 */ 4679 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx); 4680 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), 4681 pwr_mode->lane_rx); 4682 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 4683 pwr_mode->pwr_rx == FAST_MODE) 4684 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true); 4685 else 4686 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false); 4687 4688 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx); 4689 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), 4690 pwr_mode->lane_tx); 4691 if (pwr_mode->pwr_tx == FASTAUTO_MODE || 4692 pwr_mode->pwr_tx == FAST_MODE) 4693 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true); 4694 else 4695 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false); 4696 4697 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 4698 pwr_mode->pwr_tx == FASTAUTO_MODE || 4699 pwr_mode->pwr_rx == FAST_MODE || 4700 pwr_mode->pwr_tx == FAST_MODE) 4701 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), 4702 pwr_mode->hs_rate); 4703 4704 if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) { 4705 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 4706 DL_FC0ProtectionTimeOutVal_Default); 4707 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 4708 DL_TC0ReplayTimeOutVal_Default); 4709 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 4710 DL_AFC0ReqTimeOutVal_Default); 4711 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3), 4712 DL_FC1ProtectionTimeOutVal_Default); 4713 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4), 4714 DL_TC1ReplayTimeOutVal_Default); 4715 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5), 4716 DL_AFC1ReqTimeOutVal_Default); 4717 4718 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal), 4719 DL_FC0ProtectionTimeOutVal_Default); 4720 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal), 4721 DL_TC0ReplayTimeOutVal_Default); 4722 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal), 4723 DL_AFC0ReqTimeOutVal_Default); 4724 } 4725 4726 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 4727 | pwr_mode->pwr_tx); 4728 4729 if (ret) { 4730 dev_err(hba->dev, 4731 "%s: power mode change failed %d\n", __func__, ret); 4732 } else { 4733 memcpy(&hba->pwr_info, pwr_mode, 4734 sizeof(struct ufs_pa_layer_attr)); 4735 } 4736 4737 return ret; 4738} 4739 4740/** 4741 * ufshcd_config_pwr_mode - configure a new power mode 4742 * @hba: per-adapter instance 4743 * @desired_pwr_mode: desired power configuration 4744 * 4745 * Return: 0 upon success; < 0 upon failure. 4746 */ 4747int ufshcd_config_pwr_mode(struct ufs_hba *hba, 4748 struct ufs_pa_layer_attr *desired_pwr_mode) 4749{ 4750 struct ufs_pa_layer_attr final_params = { 0 }; 4751 int ret; 4752 4753 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE, 4754 desired_pwr_mode, &final_params); 4755 4756 if (ret) 4757 memcpy(&final_params, desired_pwr_mode, sizeof(final_params)); 4758 4759 ret = ufshcd_change_power_mode(hba, &final_params); 4760 4761 if (!ret) 4762 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL, 4763 &final_params); 4764 4765 return ret; 4766} 4767EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); 4768 4769/** 4770 * ufshcd_complete_dev_init() - checks device readiness 4771 * @hba: per-adapter instance 4772 * 4773 * Set fDeviceInit flag and poll until device toggles it. 4774 * 4775 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 4776 * < 0 if another error occurred. 4777 */ 4778static int ufshcd_complete_dev_init(struct ufs_hba *hba) 4779{ 4780 int err; 4781 bool flag_res = true; 4782 ktime_t timeout; 4783 4784 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, 4785 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL); 4786 if (err) { 4787 dev_err(hba->dev, 4788 "%s: setting fDeviceInit flag failed with error %d\n", 4789 __func__, err); 4790 goto out; 4791 } 4792 4793 /* Poll fDeviceInit flag to be cleared */ 4794 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT); 4795 do { 4796 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, 4797 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res); 4798 if (!flag_res) 4799 break; 4800 usleep_range(500, 1000); 4801 } while (ktime_before(ktime_get(), timeout)); 4802 4803 if (err) { 4804 dev_err(hba->dev, 4805 "%s: reading fDeviceInit flag failed with error %d\n", 4806 __func__, err); 4807 } else if (flag_res) { 4808 dev_err(hba->dev, 4809 "%s: fDeviceInit was not cleared by the device\n", 4810 __func__); 4811 err = -EBUSY; 4812 } 4813out: 4814 return err; 4815} 4816 4817/** 4818 * ufshcd_make_hba_operational - Make UFS controller operational 4819 * @hba: per adapter instance 4820 * 4821 * To bring UFS host controller to operational state, 4822 * 1. Enable required interrupts 4823 * 2. Configure interrupt aggregation 4824 * 3. Program UTRL and UTMRL base address 4825 * 4. Configure run-stop-registers 4826 * 4827 * Return: 0 if successful; < 0 upon failure. 4828 */ 4829int ufshcd_make_hba_operational(struct ufs_hba *hba) 4830{ 4831 int err = 0; 4832 u32 reg; 4833 4834 /* Enable required interrupts */ 4835 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS); 4836 4837 /* Configure interrupt aggregation */ 4838 if (ufshcd_is_intr_aggr_allowed(hba)) 4839 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO); 4840 else 4841 ufshcd_disable_intr_aggr(hba); 4842 4843 /* Configure UTRL and UTMRL base address registers */ 4844 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 4845 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 4846 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 4847 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 4848 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 4849 REG_UTP_TASK_REQ_LIST_BASE_L); 4850 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 4851 REG_UTP_TASK_REQ_LIST_BASE_H); 4852 4853 /* 4854 * UCRDY, UTMRLDY and UTRLRDY bits must be 1 4855 */ 4856 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); 4857 if (!(ufshcd_get_lists_status(reg))) { 4858 ufshcd_enable_run_stop_reg(hba); 4859 } else { 4860 dev_err(hba->dev, 4861 "Host controller not ready to process requests"); 4862 err = -EIO; 4863 } 4864 4865 return err; 4866} 4867EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational); 4868 4869/** 4870 * ufshcd_hba_stop - Send controller to reset state 4871 * @hba: per adapter instance 4872 */ 4873void ufshcd_hba_stop(struct ufs_hba *hba) 4874{ 4875 int err; 4876 4877 ufshcd_disable_irq(hba); 4878 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE); 4879 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, 4880 CONTROLLER_ENABLE, CONTROLLER_DISABLE, 4881 10, 1); 4882 ufshcd_enable_irq(hba); 4883 if (err) 4884 dev_err(hba->dev, "%s: Controller disable failed\n", __func__); 4885} 4886EXPORT_SYMBOL_GPL(ufshcd_hba_stop); 4887 4888/** 4889 * ufshcd_hba_execute_hce - initialize the controller 4890 * @hba: per adapter instance 4891 * 4892 * The controller resets itself and controller firmware initialization 4893 * sequence kicks off. When controller is ready it will set 4894 * the Host Controller Enable bit to 1. 4895 * 4896 * Return: 0 on success, non-zero value on failure. 4897 */ 4898static int ufshcd_hba_execute_hce(struct ufs_hba *hba) 4899{ 4900 int retry; 4901 4902 for (retry = 3; retry > 0; retry--) { 4903 if (ufshcd_is_hba_active(hba)) 4904 /* change controller state to "reset state" */ 4905 ufshcd_hba_stop(hba); 4906 4907 /* UniPro link is disabled at this point */ 4908 ufshcd_set_link_off(hba); 4909 4910 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); 4911 4912 /* start controller initialization sequence */ 4913 ufshcd_hba_start(hba); 4914 4915 /* 4916 * To initialize a UFS host controller HCE bit must be set to 1. 4917 * During initialization the HCE bit value changes from 1->0->1. 4918 * When the host controller completes initialization sequence 4919 * it sets the value of HCE bit to 1. The same HCE bit is read back 4920 * to check if the controller has completed initialization sequence. 4921 * So without this delay the value HCE = 1, set in the previous 4922 * instruction might be read back. 4923 * This delay can be changed based on the controller. 4924 */ 4925 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100); 4926 4927 /* wait for the host controller to complete initialization */ 4928 if (!ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, CONTROLLER_ENABLE, 4929 CONTROLLER_ENABLE, 1000, 50)) 4930 break; 4931 4932 dev_err(hba->dev, "Enabling the controller failed\n"); 4933 } 4934 4935 if (!retry) 4936 return -EIO; 4937 4938 /* enable UIC related interrupts */ 4939 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 4940 4941 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); 4942 4943 return 0; 4944} 4945 4946int ufshcd_hba_enable(struct ufs_hba *hba) 4947{ 4948 int ret; 4949 4950 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) { 4951 ufshcd_set_link_off(hba); 4952 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); 4953 4954 /* enable UIC related interrupts */ 4955 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 4956 ret = ufshcd_dme_reset(hba); 4957 if (ret) { 4958 dev_err(hba->dev, "DME_RESET failed\n"); 4959 return ret; 4960 } 4961 4962 ret = ufshcd_dme_enable(hba); 4963 if (ret) { 4964 dev_err(hba->dev, "Enabling DME failed\n"); 4965 return ret; 4966 } 4967 4968 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); 4969 } else { 4970 ret = ufshcd_hba_execute_hce(hba); 4971 } 4972 4973 return ret; 4974} 4975EXPORT_SYMBOL_GPL(ufshcd_hba_enable); 4976 4977static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer) 4978{ 4979 int tx_lanes, i, err = 0; 4980 4981 if (!peer) 4982 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4983 &tx_lanes); 4984 else 4985 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4986 &tx_lanes); 4987 for (i = 0; i < tx_lanes; i++) { 4988 if (!peer) 4989 err = ufshcd_dme_set(hba, 4990 UIC_ARG_MIB_SEL(TX_LCC_ENABLE, 4991 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), 4992 0); 4993 else 4994 err = ufshcd_dme_peer_set(hba, 4995 UIC_ARG_MIB_SEL(TX_LCC_ENABLE, 4996 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), 4997 0); 4998 if (err) { 4999 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d", 5000 __func__, peer, i, err); 5001 break; 5002 } 5003 } 5004 5005 return err; 5006} 5007 5008static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba) 5009{ 5010 return ufshcd_disable_tx_lcc(hba, true); 5011} 5012 5013void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val) 5014{ 5015 struct ufs_event_hist *e; 5016 5017 if (id >= UFS_EVT_CNT) 5018 return; 5019 5020 e = &hba->ufs_stats.event[id]; 5021 e->val[e->pos] = val; 5022 e->tstamp[e->pos] = local_clock(); 5023 e->cnt += 1; 5024 e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH; 5025 5026 ufshcd_vops_event_notify(hba, id, &val); 5027} 5028EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist); 5029 5030/** 5031 * ufshcd_link_startup - Initialize unipro link startup 5032 * @hba: per adapter instance 5033 * 5034 * Return: 0 for success, non-zero in case of failure. 5035 */ 5036static int ufshcd_link_startup(struct ufs_hba *hba) 5037{ 5038 int ret; 5039 int retries = DME_LINKSTARTUP_RETRIES; 5040 bool link_startup_again = false; 5041 5042 /* 5043 * If UFS device isn't active then we will have to issue link startup 5044 * 2 times to make sure the device state move to active. 5045 */ 5046 if (!(hba->quirks & UFSHCD_QUIRK_PERFORM_LINK_STARTUP_ONCE) && 5047 !ufshcd_is_ufs_dev_active(hba)) 5048 link_startup_again = true; 5049 5050link_startup: 5051 do { 5052 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE); 5053 5054 ret = ufshcd_dme_link_startup(hba); 5055 5056 /* check if device is detected by inter-connect layer */ 5057 if (!ret && !ufshcd_is_device_present(hba)) { 5058 ufshcd_update_evt_hist(hba, 5059 UFS_EVT_LINK_STARTUP_FAIL, 5060 0); 5061 dev_err(hba->dev, "%s: Device not present\n", __func__); 5062 ret = -ENXIO; 5063 goto out; 5064 } 5065 5066 /* 5067 * DME link lost indication is only received when link is up, 5068 * but we can't be sure if the link is up until link startup 5069 * succeeds. So reset the local Uni-Pro and try again. 5070 */ 5071 if (ret && retries && ufshcd_hba_enable(hba)) { 5072 ufshcd_update_evt_hist(hba, 5073 UFS_EVT_LINK_STARTUP_FAIL, 5074 (u32)ret); 5075 goto out; 5076 } 5077 } while (ret && retries--); 5078 5079 if (ret) { 5080 /* failed to get the link up... retire */ 5081 ufshcd_update_evt_hist(hba, 5082 UFS_EVT_LINK_STARTUP_FAIL, 5083 (u32)ret); 5084 goto out; 5085 } 5086 5087 if (link_startup_again) { 5088 link_startup_again = false; 5089 retries = DME_LINKSTARTUP_RETRIES; 5090 goto link_startup; 5091 } 5092 5093 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */ 5094 ufshcd_init_pwr_info(hba); 5095 ufshcd_print_pwr_info(hba); 5096 5097 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) { 5098 ret = ufshcd_disable_device_tx_lcc(hba); 5099 if (ret) 5100 goto out; 5101 } 5102 5103 /* Include any host controller configuration via UIC commands */ 5104 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE); 5105 if (ret) 5106 goto out; 5107 5108 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */ 5109 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); 5110 ret = ufshcd_make_hba_operational(hba); 5111out: 5112 if (ret) 5113 dev_err(hba->dev, "link startup failed %d\n", ret); 5114 return ret; 5115} 5116 5117/** 5118 * ufshcd_verify_dev_init() - Verify device initialization 5119 * @hba: per-adapter instance 5120 * 5121 * Send NOP OUT UPIU and wait for NOP IN response to check whether the 5122 * device Transport Protocol (UTP) layer is ready after a reset. 5123 * If the UTP layer at the device side is not initialized, it may 5124 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT 5125 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. 5126 * 5127 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 5128 * < 0 if another error occurred. 5129 */ 5130static int ufshcd_verify_dev_init(struct ufs_hba *hba) 5131{ 5132 int err = 0; 5133 int retries; 5134 5135 ufshcd_dev_man_lock(hba); 5136 5137 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { 5138 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, 5139 hba->nop_out_timeout); 5140 5141 if (!err || err == -ETIMEDOUT) 5142 break; 5143 5144 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); 5145 } 5146 5147 ufshcd_dev_man_unlock(hba); 5148 5149 if (err) 5150 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); 5151 return err; 5152} 5153 5154/** 5155 * ufshcd_setup_links - associate link b/w device wlun and other luns 5156 * @sdev: pointer to SCSI device 5157 * @hba: pointer to ufs hba 5158 */ 5159static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev) 5160{ 5161 struct device_link *link; 5162 5163 /* 5164 * Device wlun is the supplier & rest of the luns are consumers. 5165 * This ensures that device wlun suspends after all other luns. 5166 */ 5167 if (hba->ufs_device_wlun) { 5168 link = device_link_add(&sdev->sdev_gendev, 5169 &hba->ufs_device_wlun->sdev_gendev, 5170 DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); 5171 if (!link) { 5172 dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n", 5173 dev_name(&hba->ufs_device_wlun->sdev_gendev)); 5174 return; 5175 } 5176 hba->luns_avail--; 5177 /* Ignore REPORT_LUN wlun probing */ 5178 if (hba->luns_avail == 1) { 5179 ufshcd_rpm_put(hba); 5180 return; 5181 } 5182 } else { 5183 /* 5184 * Device wlun is probed. The assumption is that WLUNs are 5185 * scanned before other LUNs. 5186 */ 5187 hba->luns_avail--; 5188 } 5189} 5190 5191/** 5192 * ufshcd_lu_init - Initialize the relevant parameters of the LU 5193 * @hba: per-adapter instance 5194 * @sdev: pointer to SCSI device 5195 */ 5196static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev) 5197{ 5198 int len = QUERY_DESC_MAX_SIZE; 5199 u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun); 5200 u8 lun_qdepth = hba->nutrs; 5201 u8 *desc_buf; 5202 int ret; 5203 5204 desc_buf = kzalloc(len, GFP_KERNEL); 5205 if (!desc_buf) 5206 goto set_qdepth; 5207 5208 ret = ufshcd_read_unit_desc_param(hba, lun, 0, desc_buf, len); 5209 if (ret < 0) { 5210 if (ret == -EOPNOTSUPP) 5211 /* If LU doesn't support unit descriptor, its queue depth is set to 1 */ 5212 lun_qdepth = 1; 5213 kfree(desc_buf); 5214 goto set_qdepth; 5215 } 5216 5217 if (desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]) { 5218 /* 5219 * In per-LU queueing architecture, bLUQueueDepth will not be 0, then we will 5220 * use the smaller between UFSHCI CAP.NUTRS and UFS LU bLUQueueDepth 5221 */ 5222 lun_qdepth = min_t(int, desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH], hba->nutrs); 5223 } 5224 /* 5225 * According to UFS device specification, the write protection mode is only supported by 5226 * normal LU, not supported by WLUN. 5227 */ 5228 if (hba->dev_info.f_power_on_wp_en && lun < hba->dev_info.max_lu_supported && 5229 !hba->dev_info.is_lu_power_on_wp && 5230 desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP) 5231 hba->dev_info.is_lu_power_on_wp = true; 5232 5233 /* In case of RPMB LU, check if advanced RPMB mode is enabled, and get region size */ 5234 if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN) { 5235 if (desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4)) 5236 hba->dev_info.b_advanced_rpmb_en = true; 5237 hba->dev_info.rpmb_region_size[0] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION0_SIZE]; 5238 hba->dev_info.rpmb_region_size[1] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION1_SIZE]; 5239 hba->dev_info.rpmb_region_size[2] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION2_SIZE]; 5240 hba->dev_info.rpmb_region_size[3] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION3_SIZE]; 5241 5242 if (hba->dev_info.wspecversion <= 0x0220) { 5243 /* 5244 * These older spec chips have only one RPMB region, 5245 * sized between 128 kB minimum and 16 MB maximum. 5246 * No per region size fields are provided (respective 5247 * REGIONX_SIZE fields always contain zeros), so get 5248 * it from the logical block count and size fields for 5249 * compatibility 5250 * 5251 * (See JESD220C-2_2 Section 14.1.4.6 5252 * RPMB Unit Descriptor,* offset 13h, 4 bytes) 5253 */ 5254 hba->dev_info.rpmb_region_size[0] = 5255 (get_unaligned_be64(desc_buf 5256 + RPMB_UNIT_DESC_PARAM_LOGICAL_BLK_COUNT) 5257 << desc_buf[RPMB_UNIT_DESC_PARAM_LOGICAL_BLK_SIZE]) 5258 / SZ_128K; 5259 } 5260 } 5261 5262 5263 kfree(desc_buf); 5264set_qdepth: 5265 /* 5266 * For WLUNs that don't support unit descriptor, queue depth is set to 1. For LUs whose 5267 * bLUQueueDepth == 0, the queue depth is set to a maximum value that host can queue. 5268 */ 5269 dev_dbg(hba->dev, "Set LU %x queue depth %d\n", lun, lun_qdepth); 5270 scsi_change_queue_depth(sdev, lun_qdepth); 5271} 5272 5273/** 5274 * ufshcd_sdev_init - handle initial SCSI device configurations 5275 * @sdev: pointer to SCSI device 5276 * 5277 * Return: success. 5278 */ 5279static int ufshcd_sdev_init(struct scsi_device *sdev) 5280{ 5281 struct ufs_hba *hba; 5282 5283 hba = shost_priv(sdev->host); 5284 5285 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */ 5286 sdev->use_10_for_ms = 1; 5287 5288 /* DBD field should be set to 1 in mode sense(10) */ 5289 sdev->set_dbd_for_ms = 1; 5290 5291 /* allow SCSI layer to restart the device in case of errors */ 5292 sdev->allow_restart = 1; 5293 5294 /* REPORT SUPPORTED OPERATION CODES is not supported */ 5295 sdev->no_report_opcodes = 1; 5296 5297 /* WRITE_SAME command is not supported */ 5298 sdev->no_write_same = 1; 5299 5300 ufshcd_lu_init(hba, sdev); 5301 5302 ufshcd_setup_links(hba, sdev); 5303 5304 return 0; 5305} 5306 5307/** 5308 * ufshcd_change_queue_depth - change queue depth 5309 * @sdev: pointer to SCSI device 5310 * @depth: required depth to set 5311 * 5312 * Change queue depth and make sure the max. limits are not crossed. 5313 * 5314 * Return: new queue depth. 5315 */ 5316static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth) 5317{ 5318 return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue)); 5319} 5320 5321/** 5322 * ufshcd_sdev_configure - adjust SCSI device configurations 5323 * @sdev: pointer to SCSI device 5324 * @lim: queue limits 5325 * 5326 * Return: 0 (success). 5327 */ 5328static int ufshcd_sdev_configure(struct scsi_device *sdev, 5329 struct queue_limits *lim) 5330{ 5331 struct ufs_hba *hba = shost_priv(sdev->host); 5332 struct request_queue *q = sdev->request_queue; 5333 5334 lim->dma_pad_mask = PRDT_DATA_BYTE_COUNT_PAD - 1; 5335 5336 /* 5337 * Block runtime-pm until all consumers are added. 5338 * Refer ufshcd_setup_links(). 5339 */ 5340 if (is_device_wlun(sdev)) 5341 pm_runtime_get_noresume(&sdev->sdev_gendev); 5342 else if (ufshcd_is_rpm_autosuspend_allowed(hba)) 5343 sdev->rpm_autosuspend = 1; 5344 /* 5345 * Do not print messages during runtime PM to avoid never-ending cycles 5346 * of messages written back to storage by user space causing runtime 5347 * resume, causing more messages and so on. 5348 */ 5349 sdev->silence_suspend = 1; 5350 5351 if (hba->vops && hba->vops->config_scsi_dev) 5352 hba->vops->config_scsi_dev(sdev); 5353 5354 ufshcd_crypto_register(hba, q); 5355 5356 return 0; 5357} 5358 5359/** 5360 * ufshcd_sdev_destroy - remove SCSI device configurations 5361 * @sdev: pointer to SCSI device 5362 */ 5363static void ufshcd_sdev_destroy(struct scsi_device *sdev) 5364{ 5365 struct ufs_hba *hba; 5366 unsigned long flags; 5367 5368 hba = shost_priv(sdev->host); 5369 5370 /* Drop the reference as it won't be needed anymore */ 5371 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) { 5372 spin_lock_irqsave(hba->host->host_lock, flags); 5373 hba->ufs_device_wlun = NULL; 5374 spin_unlock_irqrestore(hba->host->host_lock, flags); 5375 } else if (hba->ufs_device_wlun) { 5376 struct device *supplier = NULL; 5377 5378 /* Ensure UFS Device WLUN exists and does not disappear */ 5379 spin_lock_irqsave(hba->host->host_lock, flags); 5380 if (hba->ufs_device_wlun) { 5381 supplier = &hba->ufs_device_wlun->sdev_gendev; 5382 get_device(supplier); 5383 } 5384 spin_unlock_irqrestore(hba->host->host_lock, flags); 5385 5386 if (supplier) { 5387 /* 5388 * If a LUN fails to probe (e.g. absent BOOT WLUN), the 5389 * device will not have been registered but can still 5390 * have a device link holding a reference to the device. 5391 */ 5392 device_link_remove(&sdev->sdev_gendev, supplier); 5393 put_device(supplier); 5394 } 5395 } 5396} 5397 5398/** 5399 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status 5400 * @cmd: SCSI command 5401 * @scsi_status: SCSI command status 5402 * 5403 * Return: value base on SCSI command status. 5404 */ 5405static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status) 5406{ 5407 int result = 0; 5408 5409 switch (scsi_status) { 5410 case SAM_STAT_CHECK_CONDITION: 5411 ufshcd_copy_sense_data(cmd); 5412 fallthrough; 5413 case SAM_STAT_GOOD: 5414 result |= DID_OK << 16 | scsi_status; 5415 break; 5416 case SAM_STAT_TASK_SET_FULL: 5417 case SAM_STAT_BUSY: 5418 case SAM_STAT_TASK_ABORTED: 5419 ufshcd_copy_sense_data(cmd); 5420 result |= scsi_status; 5421 break; 5422 default: 5423 result |= DID_ERROR << 16; 5424 break; 5425 } /* end of switch */ 5426 5427 return result; 5428} 5429 5430/** 5431 * ufshcd_transfer_rsp_status - Get overall status of the response 5432 * @hba: per adapter instance 5433 * @cmd: SCSI command 5434 * @cqe: pointer to the completion queue entry 5435 * 5436 * Return: result of the command to notify SCSI midlayer. 5437 */ 5438static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, 5439 struct scsi_cmnd *cmd, 5440 struct cq_entry *cqe) 5441{ 5442 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 5443 const int tag = scsi_cmd_to_rq(cmd)->tag; 5444 int result = 0; 5445 int scsi_status; 5446 enum utp_ocs ocs; 5447 u8 upiu_flags; 5448 u32 resid; 5449 5450 upiu_flags = lrbp->ucd_rsp_ptr->header.flags; 5451 resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count); 5452 /* 5453 * Test !overflow instead of underflow to support UFS devices that do 5454 * not set either flag. 5455 */ 5456 if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) 5457 scsi_set_resid(cmd, resid); 5458 5459 /* overall command status of utrd */ 5460 ocs = ufshcd_get_tr_ocs(lrbp, cqe); 5461 5462 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) { 5463 if (lrbp->ucd_rsp_ptr->header.response || 5464 lrbp->ucd_rsp_ptr->header.status) 5465 ocs = OCS_SUCCESS; 5466 } 5467 5468 switch (ocs) { 5469 case OCS_SUCCESS: 5470 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 5471 switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) { 5472 case UPIU_TRANSACTION_RESPONSE: 5473 /* 5474 * get the result based on SCSI status response 5475 * to notify the SCSI midlayer of the command status 5476 */ 5477 scsi_status = lrbp->ucd_rsp_ptr->header.status; 5478 result = ufshcd_scsi_cmd_status(cmd, scsi_status); 5479 5480 /* 5481 * Currently we are only supporting BKOPs exception 5482 * events hence we can ignore BKOPs exception event 5483 * during power management callbacks. BKOPs exception 5484 * event is not expected to be raised in runtime suspend 5485 * callback as it allows the urgent bkops. 5486 * During system suspend, we are anyway forcefully 5487 * disabling the bkops and if urgent bkops is needed 5488 * it will be enabled on system resume. Long term 5489 * solution could be to abort the system suspend if 5490 * UFS device needs urgent BKOPs. 5491 */ 5492 if (!hba->pm_op_in_progress && 5493 !ufshcd_eh_in_progress(hba) && 5494 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) 5495 /* Flushed in suspend */ 5496 schedule_work(&hba->eeh_work); 5497 break; 5498 case UPIU_TRANSACTION_REJECT_UPIU: 5499 /* TODO: handle Reject UPIU Response */ 5500 result = DID_ERROR << 16; 5501 dev_err(hba->dev, 5502 "Reject UPIU not fully implemented\n"); 5503 break; 5504 default: 5505 dev_err(hba->dev, 5506 "Unexpected request response code = %x\n", 5507 result); 5508 result = DID_ERROR << 16; 5509 break; 5510 } 5511 break; 5512 case OCS_ABORTED: 5513 case OCS_INVALID_COMMAND_STATUS: 5514 result |= DID_REQUEUE << 16; 5515 dev_warn(hba->dev, "OCS %s from controller for tag %d\n", 5516 ocs == OCS_ABORTED ? "aborted" : "invalid", tag); 5517 break; 5518 case OCS_INVALID_CMD_TABLE_ATTR: 5519 case OCS_INVALID_PRDT_ATTR: 5520 case OCS_MISMATCH_DATA_BUF_SIZE: 5521 case OCS_MISMATCH_RESP_UPIU_SIZE: 5522 case OCS_PEER_COMM_FAILURE: 5523 case OCS_FATAL_ERROR: 5524 case OCS_DEVICE_FATAL_ERROR: 5525 case OCS_INVALID_CRYPTO_CONFIG: 5526 case OCS_GENERAL_CRYPTO_ERROR: 5527 default: 5528 result |= DID_ERROR << 16; 5529 dev_err(hba->dev, "OCS error from controller = %x for tag %d\n", 5530 ocs, tag); 5531 ufshcd_print_evt_hist(hba); 5532 ufshcd_print_host_state(hba); 5533 break; 5534 } /* end of switch */ 5535 5536 if ((host_byte(result) != DID_OK) && 5537 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) { 5538 if (cqe) 5539 ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry)); 5540 ufshcd_print_tr(hba, cmd, true); 5541 } 5542 return result; 5543} 5544 5545static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba, 5546 u32 intr_mask) 5547{ 5548 if (!ufshcd_is_auto_hibern8_supported(hba) || 5549 !ufshcd_is_auto_hibern8_enabled(hba)) 5550 return false; 5551 5552 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK)) 5553 return false; 5554 5555 if (hba->active_uic_cmd && 5556 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER || 5557 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT)) 5558 return false; 5559 5560 return true; 5561} 5562 5563/** 5564 * ufshcd_uic_cmd_compl - handle completion of uic command 5565 * @hba: per adapter instance 5566 * @intr_status: interrupt status generated by the controller 5567 * 5568 * Return: 5569 * IRQ_HANDLED - If interrupt is valid 5570 * IRQ_NONE - If invalid interrupt 5571 */ 5572static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) 5573{ 5574 irqreturn_t retval = IRQ_NONE; 5575 struct uic_command *cmd; 5576 5577 guard(spinlock_irqsave)(hba->host->host_lock); 5578 cmd = hba->active_uic_cmd; 5579 if (!cmd) 5580 return retval; 5581 5582 if (ufshcd_is_auto_hibern8_error(hba, intr_status)) 5583 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status); 5584 5585 if (intr_status & UIC_COMMAND_COMPL) { 5586 cmd->argument2 |= ufshcd_get_uic_cmd_result(hba); 5587 cmd->argument3 = ufshcd_get_dme_attr_val(hba); 5588 if (!hba->uic_async_done) 5589 cmd->cmd_active = false; 5590 complete(&cmd->done); 5591 retval = IRQ_HANDLED; 5592 } 5593 5594 if (intr_status & UFSHCD_UIC_PWR_MASK && hba->uic_async_done) { 5595 cmd->cmd_active = false; 5596 complete(hba->uic_async_done); 5597 retval = IRQ_HANDLED; 5598 } 5599 5600 if (retval == IRQ_HANDLED) 5601 ufshcd_add_uic_command_trace(hba, cmd, UFS_CMD_COMP); 5602 5603 return retval; 5604} 5605 5606/* Release the resources allocated for processing a SCSI command. */ 5607void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) 5608{ 5609 scsi_dma_unmap(cmd); 5610 ufshcd_crypto_clear_prdt(hba, cmd); 5611 ufshcd_release(hba); 5612 ufshcd_clk_scaling_update_busy(hba); 5613} 5614 5615/** 5616 * ufshcd_compl_one_cqe - handle a completion queue entry 5617 * @hba: per adapter instance 5618 * @task_tag: the task tag of the request to be completed 5619 * @cqe: pointer to the completion queue entry 5620 */ 5621void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, 5622 struct cq_entry *cqe) 5623{ 5624 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 5625 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 5626 enum utp_ocs ocs; 5627 5628 if (WARN_ONCE(!cmd, "cqe->command_desc_base_addr = %#llx\n", 5629 le64_to_cpu(cqe->command_desc_base_addr))) 5630 return; 5631 5632 if (hba->monitor.enabled) { 5633 lrbp->compl_time_stamp = ktime_get(); 5634 lrbp->compl_time_stamp_local_clock = local_clock(); 5635 } 5636 if (ufshcd_is_scsi_cmd(cmd)) { 5637 if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) 5638 ufshcd_update_monitor(hba, cmd); 5639 ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); 5640 cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe); 5641 ufshcd_release_scsi_cmd(hba, cmd); 5642 } else { 5643 if (cqe) { 5644 ocs = cqe->overall_status & MASK_OCS; 5645 lrbp->utr_descriptor_ptr->header.ocs = ocs; 5646 } else { 5647 ocs = lrbp->utr_descriptor_ptr->header.ocs; 5648 } 5649 ufshcd_add_query_upiu_trace( 5650 hba, 5651 ocs == OCS_SUCCESS ? UFS_QUERY_COMP : UFS_QUERY_ERR, 5652 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); 5653 cmd->result = 0; 5654 } 5655 /* Do not touch lrbp after scsi_done() has been called. */ 5656 scsi_done(cmd); 5657} 5658 5659/** 5660 * __ufshcd_transfer_req_compl - handle SCSI and query command completion 5661 * @hba: per adapter instance 5662 * @completed_reqs: bitmask that indicates which requests to complete 5663 */ 5664static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, 5665 unsigned long completed_reqs) 5666{ 5667 int tag; 5668 5669 for_each_set_bit(tag, &completed_reqs, hba->nutrs) 5670 ufshcd_compl_one_cqe(hba, tag, NULL); 5671} 5672 5673/* Any value that is not an existing queue number is fine for this constant. */ 5674enum { 5675 UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1 5676}; 5677 5678static void ufshcd_clear_polled(struct ufs_hba *hba, 5679 unsigned long *completed_reqs) 5680{ 5681 int tag; 5682 5683 for_each_set_bit(tag, completed_reqs, hba->nutrs) { 5684 struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag); 5685 5686 if (!cmd) 5687 continue; 5688 if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED) 5689 __clear_bit(tag, completed_reqs); 5690 } 5691} 5692 5693/* 5694 * Return: > 0 if one or more commands have been completed or 0 if no 5695 * requests have been completed. 5696 */ 5697static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) 5698{ 5699 struct ufs_hba *hba = shost_priv(shost); 5700 unsigned long completed_reqs, flags; 5701 u32 tr_doorbell; 5702 struct ufs_hw_queue *hwq; 5703 5704 if (hba->mcq_enabled) { 5705 hwq = &hba->uhq[queue_num]; 5706 5707 return ufshcd_mcq_poll_cqe_lock(hba, hwq); 5708 } 5709 5710 spin_lock_irqsave(&hba->outstanding_lock, flags); 5711 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 5712 completed_reqs = ~tr_doorbell & hba->outstanding_reqs; 5713 WARN_ONCE(completed_reqs & ~hba->outstanding_reqs, 5714 "completed: %#lx; outstanding: %#lx\n", completed_reqs, 5715 hba->outstanding_reqs); 5716 if (queue_num == UFSHCD_POLL_FROM_INTERRUPT_CONTEXT) { 5717 /* Do not complete polled requests from interrupt context. */ 5718 ufshcd_clear_polled(hba, &completed_reqs); 5719 } 5720 hba->outstanding_reqs &= ~completed_reqs; 5721 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 5722 5723 if (completed_reqs) 5724 __ufshcd_transfer_req_compl(hba, completed_reqs); 5725 5726 return completed_reqs != 0; 5727} 5728 5729static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) 5730{ 5731 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 5732 struct scsi_device *sdev = rq->q->queuedata; 5733 struct Scsi_Host *shost = sdev->host; 5734 struct ufs_hba *hba = shost_priv(shost); 5735 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 5736 5737 if (blk_mq_is_reserved_rq(rq) || !hwq) 5738 return true; 5739 5740 ufshcd_mcq_compl_all_cqes_lock(hba, hwq); 5741 5742 /* 5743 * For those cmds of which the cqes are not present in the cq, complete 5744 * them explicitly. 5745 */ 5746 scoped_guard(spinlock_irqsave, &hwq->cq_lock) { 5747 if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { 5748 set_host_byte(cmd, DID_REQUEUE); 5749 ufshcd_release_scsi_cmd(hba, cmd); 5750 scsi_done(cmd); 5751 } 5752 } 5753 5754 return true; 5755} 5756 5757static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) 5758{ 5759 struct scsi_device *sdev = rq->q->queuedata; 5760 struct Scsi_Host *shost = sdev->host; 5761 struct ufs_hba *hba = shost_priv(shost); 5762 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); 5763 5764 if (!blk_mq_is_reserved_rq(rq) && hwq) 5765 ufshcd_mcq_poll_cqe_lock(hba, hwq); 5766 5767 return true; 5768} 5769 5770/** 5771 * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is 5772 * invoked from the error handler context or ufshcd_host_reset_and_restore() 5773 * to complete the pending transfers and free the resources associated with 5774 * the scsi command. 5775 * 5776 * @hba: per adapter instance 5777 * @force_compl: This flag is set to true when invoked 5778 * from ufshcd_host_reset_and_restore() in which case it requires special 5779 * handling because the host controller has been reset by ufshcd_hba_stop(). 5780 */ 5781static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, 5782 bool force_compl) 5783{ 5784 blk_mq_tagset_busy_iter(&hba->host->tag_set, 5785 force_compl ? ufshcd_mcq_force_compl_one : 5786 ufshcd_mcq_compl_one, 5787 NULL); 5788} 5789 5790/** 5791 * ufshcd_transfer_req_compl - handle SCSI and query command completion 5792 * @hba: per adapter instance 5793 * 5794 * Return: 5795 * IRQ_HANDLED - If interrupt is valid 5796 * IRQ_NONE - If invalid interrupt 5797 */ 5798static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba) 5799{ 5800 /* Resetting interrupt aggregation counters first and reading the 5801 * DOOR_BELL afterward allows us to handle all the completed requests. 5802 * In order to prevent other interrupts starvation the DB is read once 5803 * after reset. The down side of this solution is the possibility of 5804 * false interrupt if device completes another request after resetting 5805 * aggregation and before reading the DB. 5806 */ 5807 if (ufshcd_is_intr_aggr_allowed(hba) && 5808 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR)) 5809 ufshcd_reset_intr_aggr(hba); 5810 5811 if (ufs_fail_completion(hba)) 5812 return IRQ_HANDLED; 5813 5814 /* 5815 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we 5816 * do not want polling to trigger spurious interrupt complaints. 5817 */ 5818 ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT); 5819 5820 return IRQ_HANDLED; 5821} 5822 5823int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask) 5824{ 5825 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 5826 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, 5827 &ee_ctrl_mask); 5828} 5829 5830int ufshcd_write_ee_control(struct ufs_hba *hba) 5831{ 5832 int err; 5833 5834 mutex_lock(&hba->ee_ctrl_mutex); 5835 err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask); 5836 mutex_unlock(&hba->ee_ctrl_mutex); 5837 if (err) 5838 dev_err(hba->dev, "%s: failed to write ee control %d\n", 5839 __func__, err); 5840 return err; 5841} 5842 5843int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, 5844 const u16 *other_mask, u16 set, u16 clr) 5845{ 5846 u16 new_mask, ee_ctrl_mask; 5847 int err = 0; 5848 5849 mutex_lock(&hba->ee_ctrl_mutex); 5850 new_mask = (*mask & ~clr) | set; 5851 ee_ctrl_mask = new_mask | *other_mask; 5852 if (ee_ctrl_mask != hba->ee_ctrl_mask) 5853 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask); 5854 /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */ 5855 if (!err) { 5856 hba->ee_ctrl_mask = ee_ctrl_mask; 5857 *mask = new_mask; 5858 } 5859 mutex_unlock(&hba->ee_ctrl_mutex); 5860 return err; 5861} 5862 5863/** 5864 * ufshcd_disable_ee - disable exception event 5865 * @hba: per-adapter instance 5866 * @mask: exception event to disable 5867 * 5868 * Disables exception event in the device so that the EVENT_ALERT 5869 * bit is not set. 5870 * 5871 * Return: zero on success, non-zero error value on failure. 5872 */ 5873static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) 5874{ 5875 return ufshcd_update_ee_drv_mask(hba, 0, mask); 5876} 5877 5878/** 5879 * ufshcd_enable_ee - enable exception event 5880 * @hba: per-adapter instance 5881 * @mask: exception event to enable 5882 * 5883 * Enable corresponding exception event in the device to allow 5884 * device to alert host in critical scenarios. 5885 * 5886 * Return: zero on success, non-zero error value on failure. 5887 */ 5888static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) 5889{ 5890 return ufshcd_update_ee_drv_mask(hba, mask, 0); 5891} 5892 5893/** 5894 * ufshcd_enable_auto_bkops - Allow device managed BKOPS 5895 * @hba: per-adapter instance 5896 * 5897 * Allow device to manage background operations on its own. Enabling 5898 * this might lead to inconsistent latencies during normal data transfers 5899 * as the device is allowed to manage its own way of handling background 5900 * operations. 5901 * 5902 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 5903 * < 0 if another error occurred. 5904 */ 5905static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) 5906{ 5907 int err = 0; 5908 5909 if (hba->auto_bkops_enabled) 5910 goto out; 5911 5912 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, 5913 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL); 5914 if (err) { 5915 dev_err(hba->dev, "%s: failed to enable bkops %d\n", 5916 __func__, err); 5917 goto out; 5918 } 5919 5920 hba->auto_bkops_enabled = true; 5921 trace_ufshcd_auto_bkops_state(hba, "Enabled"); 5922 5923 /* No need of URGENT_BKOPS exception from the device */ 5924 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 5925 if (err) 5926 dev_err(hba->dev, "%s: failed to disable exception event %d\n", 5927 __func__, err); 5928out: 5929 return err; 5930} 5931 5932/** 5933 * ufshcd_disable_auto_bkops - block device in doing background operations 5934 * @hba: per-adapter instance 5935 * 5936 * Disabling background operations improves command response latency but 5937 * has drawback of device moving into critical state where the device is 5938 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the 5939 * host is idle so that BKOPS are managed effectively without any negative 5940 * impacts. 5941 * 5942 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 5943 * < 0 if another error occurred. 5944 */ 5945static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) 5946{ 5947 int err = 0; 5948 5949 if (!hba->auto_bkops_enabled) 5950 goto out; 5951 5952 /* 5953 * If host assisted BKOPs is to be enabled, make sure 5954 * urgent bkops exception is allowed. 5955 */ 5956 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS); 5957 if (err) { 5958 dev_err(hba->dev, "%s: failed to enable exception event %d\n", 5959 __func__, err); 5960 goto out; 5961 } 5962 5963 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG, 5964 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL); 5965 if (err) { 5966 dev_err(hba->dev, "%s: failed to disable bkops %d\n", 5967 __func__, err); 5968 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 5969 goto out; 5970 } 5971 5972 hba->auto_bkops_enabled = false; 5973 trace_ufshcd_auto_bkops_state(hba, "Disabled"); 5974 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; 5975 hba->is_urgent_bkops_lvl_checked = false; 5976out: 5977 return err; 5978} 5979 5980/** 5981 * ufshcd_force_reset_auto_bkops - force reset auto bkops state 5982 * @hba: per adapter instance 5983 * 5984 * After a device reset the device may toggle the BKOPS_EN flag 5985 * to default value. The s/w tracking variables should be updated 5986 * as well. This function would change the auto-bkops state based on 5987 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND. 5988 */ 5989static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) 5990{ 5991 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) { 5992 hba->auto_bkops_enabled = false; 5993 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; 5994 ufshcd_enable_auto_bkops(hba); 5995 } else { 5996 hba->auto_bkops_enabled = true; 5997 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS; 5998 ufshcd_disable_auto_bkops(hba); 5999 } 6000 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; 6001 hba->is_urgent_bkops_lvl_checked = false; 6002} 6003 6004static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) 6005{ 6006 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6007 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status); 6008} 6009 6010/** 6011 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status 6012 * @hba: per-adapter instance 6013 * 6014 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn 6015 * flag in the device to permit background operations if the device 6016 * bkops_status is greater than or equal to the "hba->urgent_bkops_lvl", 6017 * disable otherwise. 6018 * 6019 * Return: 0 for success, non-zero in case of failure. 6020 * 6021 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag 6022 * to know whether auto bkops is enabled or disabled after this function 6023 * returns control to it. 6024 */ 6025static int ufshcd_bkops_ctrl(struct ufs_hba *hba) 6026{ 6027 enum bkops_status status = hba->urgent_bkops_lvl; 6028 u32 curr_status = 0; 6029 int err; 6030 6031 err = ufshcd_get_bkops_status(hba, &curr_status); 6032 if (err) { 6033 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 6034 __func__, err); 6035 goto out; 6036 } else if (curr_status > BKOPS_STATUS_MAX) { 6037 dev_err(hba->dev, "%s: invalid BKOPS status %d\n", 6038 __func__, curr_status); 6039 err = -EINVAL; 6040 goto out; 6041 } 6042 6043 if (curr_status >= status) 6044 err = ufshcd_enable_auto_bkops(hba); 6045 else 6046 err = ufshcd_disable_auto_bkops(hba); 6047out: 6048 return err; 6049} 6050 6051static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) 6052{ 6053 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6054 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status); 6055} 6056 6057static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba) 6058{ 6059 int err; 6060 u32 curr_status = 0; 6061 6062 if (hba->is_urgent_bkops_lvl_checked) 6063 goto enable_auto_bkops; 6064 6065 err = ufshcd_get_bkops_status(hba, &curr_status); 6066 if (err) { 6067 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 6068 __func__, err); 6069 goto out; 6070 } 6071 6072 /* 6073 * We are seeing that some devices are raising the urgent bkops 6074 * exception events even when BKOPS status doesn't indicate performace 6075 * impacted or critical. Handle these device by determining their urgent 6076 * bkops status at runtime. 6077 */ 6078 if ((curr_status > BKOPS_STATUS_NO_OP) && (curr_status < BKOPS_STATUS_PERF_IMPACT)) { 6079 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n", 6080 __func__, curr_status); 6081 /* update the current status as the urgent bkops level */ 6082 hba->urgent_bkops_lvl = curr_status; 6083 hba->is_urgent_bkops_lvl_checked = true; 6084 } 6085 6086enable_auto_bkops: 6087 err = ufshcd_enable_auto_bkops(hba); 6088out: 6089 if (err < 0) 6090 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n", 6091 __func__, err); 6092} 6093 6094/* 6095 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 6096 * < 0 if another error occurred. 6097 */ 6098int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id) 6099{ 6100 struct utp_upiu_query_v4_0 *upiu_resp; 6101 struct ufs_query_req *request = NULL; 6102 struct ufs_query_res *response = NULL; 6103 int err; 6104 6105 if (hba->dev_info.wspecversion < 0x410) 6106 return -EOPNOTSUPP; 6107 6108 ufshcd_hold(hba); 6109 mutex_lock(&hba->dev_cmd.lock); 6110 6111 ufshcd_init_query(hba, &request, &response, 6112 UPIU_QUERY_OPCODE_READ_ATTR, 6113 QUERY_ATTR_IDN_DEV_LVL_EXCEPTION_ID, 0, 0); 6114 6115 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 6116 6117 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 6118 6119 if (err) { 6120 dev_err(hba->dev, "%s: failed to read device level exception %d\n", 6121 __func__, err); 6122 goto out; 6123 } 6124 6125 upiu_resp = (struct utp_upiu_query_v4_0 *)response; 6126 *exception_id = get_unaligned_be64(&upiu_resp->osf3); 6127out: 6128 mutex_unlock(&hba->dev_cmd.lock); 6129 ufshcd_release(hba); 6130 6131 return err; 6132} 6133 6134static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn) 6135{ 6136 u8 index; 6137 enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG : 6138 UPIU_QUERY_OPCODE_CLEAR_FLAG; 6139 6140 index = ufshcd_wb_get_query_index(hba); 6141 return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL); 6142} 6143 6144int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable) 6145{ 6146 int ret; 6147 6148 if (!ufshcd_is_wb_allowed(hba) || 6149 hba->dev_info.wb_enabled == enable) 6150 return 0; 6151 6152 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN); 6153 if (ret) { 6154 dev_err(hba->dev, "%s: Write Booster %s failed %d\n", 6155 __func__, enable ? "enabling" : "disabling", ret); 6156 return ret; 6157 } 6158 6159 hba->dev_info.wb_enabled = enable; 6160 dev_dbg(hba->dev, "%s: Write Booster %s\n", 6161 __func__, enable ? "enabled" : "disabled"); 6162 6163 return ret; 6164} 6165 6166static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, 6167 bool enable) 6168{ 6169 int ret; 6170 6171 ret = __ufshcd_wb_toggle(hba, enable, 6172 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8); 6173 if (ret) { 6174 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n", 6175 __func__, enable ? "enabling" : "disabling", ret); 6176 return; 6177 } 6178 dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n", 6179 __func__, enable ? "enabled" : "disabled"); 6180} 6181 6182int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable) 6183{ 6184 int ret; 6185 6186 if (!ufshcd_is_wb_allowed(hba) || 6187 hba->dev_info.wb_buf_flush_enabled == enable) 6188 return 0; 6189 6190 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN); 6191 if (ret) { 6192 dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n", 6193 __func__, enable ? "enabling" : "disabling", ret); 6194 return ret; 6195 } 6196 6197 hba->dev_info.wb_buf_flush_enabled = enable; 6198 dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n", 6199 __func__, enable ? "enabled" : "disabled"); 6200 6201 return ret; 6202} 6203 6204int ufshcd_wb_set_resize_en(struct ufs_hba *hba, enum wb_resize_en en_mode) 6205{ 6206 int ret; 6207 u8 index; 6208 6209 index = ufshcd_wb_get_query_index(hba); 6210 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 6211 QUERY_ATTR_IDN_WB_BUF_RESIZE_EN, index, 0, &en_mode); 6212 if (ret) 6213 dev_err(hba->dev, "%s: Enable WB buf resize operation failed %d\n", 6214 __func__, ret); 6215 6216 return ret; 6217} 6218 6219static bool ufshcd_wb_curr_buff_threshold_check(struct ufs_hba *hba, 6220 u32 avail_buf) 6221{ 6222 u32 cur_buf; 6223 int ret; 6224 u8 index; 6225 6226 index = ufshcd_wb_get_query_index(hba); 6227 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6228 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE, 6229 index, 0, &cur_buf); 6230 if (ret) { 6231 dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n", 6232 __func__, ret); 6233 return false; 6234 } 6235 6236 if (!cur_buf) { 6237 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n", 6238 cur_buf); 6239 return false; 6240 } 6241 /* Let it continue to flush when available buffer exceeds threshold */ 6242 return avail_buf < hba->vps->wb_flush_threshold; 6243} 6244 6245static void ufshcd_wb_force_disable(struct ufs_hba *hba) 6246{ 6247 if (ufshcd_is_wb_buf_flush_allowed(hba)) 6248 ufshcd_wb_toggle_buf_flush(hba, false); 6249 6250 ufshcd_wb_toggle_buf_flush_during_h8(hba, false); 6251 ufshcd_wb_toggle(hba, false); 6252 hba->caps &= ~UFSHCD_CAP_WB_EN; 6253 6254 dev_info(hba->dev, "%s: WB force disabled\n", __func__); 6255} 6256 6257static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba) 6258{ 6259 u32 lifetime; 6260 int ret; 6261 u8 index; 6262 6263 index = ufshcd_wb_get_query_index(hba); 6264 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6265 QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST, 6266 index, 0, &lifetime); 6267 if (ret) { 6268 dev_err(hba->dev, 6269 "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n", 6270 __func__, ret); 6271 return false; 6272 } 6273 6274 if (lifetime == UFS_WB_EXCEED_LIFETIME) { 6275 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n", 6276 __func__, lifetime); 6277 return false; 6278 } 6279 6280 dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n", 6281 __func__, lifetime); 6282 6283 return true; 6284} 6285 6286static bool ufshcd_wb_need_flush(struct ufs_hba *hba) 6287{ 6288 int ret; 6289 u32 avail_buf; 6290 u8 index; 6291 6292 if (!ufshcd_is_wb_allowed(hba)) 6293 return false; 6294 6295 if (!ufshcd_is_wb_buf_lifetime_available(hba)) { 6296 ufshcd_wb_force_disable(hba); 6297 return false; 6298 } 6299 6300 /* 6301 * With user-space reduction enabled, it's enough to enable flush 6302 * by checking only the available buffer. The threshold 6303 * defined here is > 90% full. 6304 * With user-space preserved enabled, the current-buffer 6305 * should be checked too because the wb buffer size can reduce 6306 * when disk tends to be full. This info is provided by current 6307 * buffer (dCurrentWriteBoosterBufferSize). 6308 */ 6309 index = ufshcd_wb_get_query_index(hba); 6310 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6311 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE, 6312 index, 0, &avail_buf); 6313 if (ret) { 6314 dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n", 6315 __func__, ret); 6316 return false; 6317 } 6318 6319 if (!hba->dev_info.b_presrv_uspc_en) 6320 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10); 6321 6322 return ufshcd_wb_curr_buff_threshold_check(hba, avail_buf); 6323} 6324 6325static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work) 6326{ 6327 struct ufs_hba *hba = container_of(to_delayed_work(work), 6328 struct ufs_hba, 6329 rpm_dev_flush_recheck_work); 6330 /* 6331 * To prevent unnecessary VCC power drain after device finishes 6332 * WriteBooster buffer flush or Auto BKOPs, force runtime resume 6333 * after a certain delay to recheck the threshold by next runtime 6334 * suspend. 6335 */ 6336 ufshcd_rpm_get_sync(hba); 6337 ufshcd_rpm_put_sync(hba); 6338} 6339 6340/** 6341 * ufshcd_exception_event_handler - handle exceptions raised by device 6342 * @work: pointer to work data 6343 * 6344 * Read bExceptionEventStatus attribute from the device and handle the 6345 * exception event accordingly. 6346 */ 6347static void ufshcd_exception_event_handler(struct work_struct *work) 6348{ 6349 struct ufs_hba *hba; 6350 int err; 6351 u32 status = 0; 6352 hba = container_of(work, struct ufs_hba, eeh_work); 6353 6354 err = ufshcd_get_ee_status(hba, &status); 6355 if (err) { 6356 dev_err(hba->dev, "%s: failed to get exception status %d\n", 6357 __func__, err); 6358 return; 6359 } 6360 6361 trace_ufshcd_exception_event(hba, status); 6362 6363 if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS) 6364 ufshcd_bkops_exception_event_handler(hba); 6365 6366 if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP) 6367 ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); 6368 6369 if (status & hba->ee_drv_mask & MASK_EE_HEALTH_CRITICAL) { 6370 hba->critical_health_count++; 6371 sysfs_notify(&hba->dev->kobj, NULL, "critical_health"); 6372 } 6373 6374 if (status & hba->ee_drv_mask & MASK_EE_DEV_LVL_EXCEPTION) { 6375 atomic_inc(&hba->dev_lvl_exception_count); 6376 sysfs_notify(&hba->dev->kobj, NULL, "device_lvl_exception_count"); 6377 } 6378 6379 ufs_debugfs_exception_event(hba, status); 6380} 6381 6382/* Complete requests that have door-bell cleared */ 6383static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl) 6384{ 6385 if (hba->mcq_enabled) 6386 ufshcd_mcq_compl_pending_transfer(hba, force_compl); 6387 else 6388 ufshcd_transfer_req_compl(hba); 6389 6390 ufshcd_tmc_handler(hba); 6391} 6392 6393/** 6394 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is 6395 * to recover from the DL NAC errors or not. 6396 * @hba: per-adapter instance 6397 * 6398 * Return: true if error handling is required, false otherwise. 6399 */ 6400static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba) 6401{ 6402 unsigned long flags; 6403 bool err_handling = true; 6404 6405 spin_lock_irqsave(hba->host->host_lock, flags); 6406 /* 6407 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the 6408 * device fatal error and/or DL NAC & REPLAY timeout errors. 6409 */ 6410 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR)) 6411 goto out; 6412 6413 if ((hba->saved_err & DEVICE_FATAL_ERROR) || 6414 ((hba->saved_err & UIC_ERROR) && 6415 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR))) 6416 goto out; 6417 6418 if ((hba->saved_err & UIC_ERROR) && 6419 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) { 6420 int err; 6421 /* 6422 * wait for 50ms to see if we can get any other errors or not. 6423 */ 6424 spin_unlock_irqrestore(hba->host->host_lock, flags); 6425 msleep(50); 6426 spin_lock_irqsave(hba->host->host_lock, flags); 6427 6428 /* 6429 * now check if we have got any other severe errors other than 6430 * DL NAC error? 6431 */ 6432 if ((hba->saved_err & INT_FATAL_ERRORS) || 6433 ((hba->saved_err & UIC_ERROR) && 6434 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR))) 6435 goto out; 6436 6437 /* 6438 * As DL NAC is the only error received so far, send out NOP 6439 * command to confirm if link is still active or not. 6440 * - If we don't get any response then do error recovery. 6441 * - If we get response then clear the DL NAC error bit. 6442 */ 6443 6444 spin_unlock_irqrestore(hba->host->host_lock, flags); 6445 err = ufshcd_verify_dev_init(hba); 6446 spin_lock_irqsave(hba->host->host_lock, flags); 6447 6448 if (err) 6449 goto out; 6450 6451 /* Link seems to be alive hence ignore the DL NAC errors */ 6452 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR) 6453 hba->saved_err &= ~UIC_ERROR; 6454 /* clear NAC error */ 6455 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR; 6456 if (!hba->saved_uic_err) 6457 err_handling = false; 6458 } 6459out: 6460 spin_unlock_irqrestore(hba->host->host_lock, flags); 6461 return err_handling; 6462} 6463 6464/* host lock must be held before calling this func */ 6465static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba) 6466{ 6467 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) || 6468 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)); 6469} 6470 6471void ufshcd_schedule_eh_work(struct ufs_hba *hba) 6472{ 6473 lockdep_assert_held(hba->host->host_lock); 6474 6475 /* handle fatal errors only when link is not in error state */ 6476 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) { 6477 if (hba->force_reset || ufshcd_is_link_broken(hba) || 6478 ufshcd_is_saved_err_fatal(hba)) 6479 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL; 6480 else 6481 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL; 6482 queue_work(hba->eh_wq, &hba->eh_work); 6483 } 6484} 6485 6486void ufshcd_force_error_recovery(struct ufs_hba *hba) 6487{ 6488 spin_lock_irq(hba->host->host_lock); 6489 hba->force_reset = true; 6490 ufshcd_schedule_eh_work(hba); 6491 spin_unlock_irq(hba->host->host_lock); 6492} 6493EXPORT_SYMBOL_GPL(ufshcd_force_error_recovery); 6494 6495static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow) 6496{ 6497 mutex_lock(&hba->wb_mutex); 6498 down_write(&hba->clk_scaling_lock); 6499 hba->clk_scaling.is_allowed = allow; 6500 up_write(&hba->clk_scaling_lock); 6501 mutex_unlock(&hba->wb_mutex); 6502} 6503 6504static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend) 6505{ 6506 if (suspend) { 6507 if (hba->clk_scaling.is_enabled) 6508 ufshcd_suspend_clkscaling(hba); 6509 ufshcd_clk_scaling_allow(hba, false); 6510 } else { 6511 ufshcd_clk_scaling_allow(hba, true); 6512 if (hba->clk_scaling.is_enabled) 6513 ufshcd_resume_clkscaling(hba); 6514 } 6515} 6516 6517static void ufshcd_err_handling_prepare(struct ufs_hba *hba) 6518{ 6519 /* 6520 * A WLUN resume failure could potentially lead to the HBA being 6521 * runtime suspended, so take an extra reference on hba->dev. 6522 */ 6523 pm_runtime_get_sync(hba->dev); 6524 ufshcd_rpm_get_sync(hba); 6525 if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) || 6526 hba->is_sys_suspended) { 6527 enum ufs_pm_op pm_op; 6528 6529 /* 6530 * Don't assume anything of resume, if 6531 * resume fails, irq and clocks can be OFF, and powers 6532 * can be OFF or in LPM. 6533 */ 6534 ufshcd_setup_hba_vreg(hba, true); 6535 ufshcd_enable_irq(hba); 6536 ufshcd_setup_vreg(hba, true); 6537 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 6538 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 6539 ufshcd_hold(hba); 6540 if (!ufshcd_is_clkgating_allowed(hba)) 6541 ufshcd_setup_clocks(hba, true); 6542 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM; 6543 ufshcd_vops_resume(hba, pm_op); 6544 } else { 6545 ufshcd_hold(hba); 6546 if (ufshcd_is_clkscaling_supported(hba) && 6547 hba->clk_scaling.is_enabled) 6548 ufshcd_suspend_clkscaling(hba); 6549 ufshcd_clk_scaling_allow(hba, false); 6550 } 6551 /* Wait for ongoing ufshcd_queuecommand() calls to finish. */ 6552 blk_mq_quiesce_tagset(&hba->host->tag_set); 6553 cancel_work_sync(&hba->eeh_work); 6554} 6555 6556static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) 6557{ 6558 blk_mq_unquiesce_tagset(&hba->host->tag_set); 6559 ufshcd_release(hba); 6560 if (ufshcd_is_clkscaling_supported(hba)) 6561 ufshcd_clk_scaling_suspend(hba, false); 6562 ufshcd_rpm_put(hba); 6563 pm_runtime_put(hba->dev); 6564} 6565 6566static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba) 6567{ 6568 return (!hba->is_powered || hba->shutting_down || 6569 !hba->ufs_device_wlun || 6570 hba->ufshcd_state == UFSHCD_STATE_ERROR || 6571 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset || 6572 ufshcd_is_link_broken(hba)))); 6573} 6574 6575#ifdef CONFIG_PM 6576static void ufshcd_recover_pm_error(struct ufs_hba *hba) 6577{ 6578 struct scsi_target *starget = hba->ufs_device_wlun->sdev_target; 6579 struct Scsi_Host *shost = hba->host; 6580 struct scsi_device *sdev; 6581 struct request_queue *q; 6582 bool resume_sdev_queues = false; 6583 6584 hba->is_sys_suspended = false; 6585 6586 /* 6587 * Ensure the parent's error status is cleared before proceeding 6588 * to the child, as the parent must be active to activate the child. 6589 */ 6590 if (hba->dev->power.runtime_error) { 6591 /* hba->dev has no functional parent thus simplily set RPM_ACTIVE */ 6592 pm_runtime_set_active(hba->dev); 6593 resume_sdev_queues = true; 6594 } 6595 6596 if (hba->ufs_device_wlun->sdev_gendev.power.runtime_error) { 6597 /* 6598 * starget, parent of wlun, might be suspended if wlun resume failed. 6599 * Make sure parent is resumed before set child (wlun) active. 6600 */ 6601 pm_runtime_get_sync(&starget->dev); 6602 pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev); 6603 pm_runtime_put_sync(&starget->dev); 6604 resume_sdev_queues = true; 6605 } 6606 6607 /* 6608 * If wlun device had runtime error, we also need to resume those 6609 * consumer scsi devices in case any of them has failed to be 6610 * resumed due to supplier runtime resume failure. This is to unblock 6611 * blk_queue_enter in case there are bios waiting inside it. 6612 */ 6613 if (resume_sdev_queues) { 6614 shost_for_each_device(sdev, shost) { 6615 q = sdev->request_queue; 6616 if (q->dev && (q->rpm_status == RPM_SUSPENDED || 6617 q->rpm_status == RPM_SUSPENDING)) 6618 pm_request_resume(q->dev); 6619 } 6620 } 6621} 6622#else 6623static inline void ufshcd_recover_pm_error(struct ufs_hba *hba) 6624{ 6625} 6626#endif 6627 6628static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba) 6629{ 6630 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info; 6631 u32 mode; 6632 6633 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode); 6634 6635 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK)) 6636 return true; 6637 6638 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK)) 6639 return true; 6640 6641 return false; 6642} 6643 6644static bool ufshcd_abort_one(struct request *rq, void *priv) 6645{ 6646 int *ret = priv; 6647 u32 tag = rq->tag; 6648 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); 6649 struct scsi_device *sdev = cmd->device; 6650 struct Scsi_Host *shost = sdev->host; 6651 struct ufs_hba *hba = shost_priv(shost); 6652 6653 if (blk_mq_is_reserved_rq(rq)) 6654 return true; 6655 6656 *ret = ufshcd_try_to_abort_task(hba, tag); 6657 dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, 6658 ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1, 6659 *ret ? "failed" : "succeeded"); 6660 6661 return *ret == 0; 6662} 6663 6664/** 6665 * ufshcd_abort_all - Abort all pending commands. 6666 * @hba: Host bus adapter pointer. 6667 * 6668 * Return: true if and only if the host controller needs to be reset. 6669 */ 6670static bool ufshcd_abort_all(struct ufs_hba *hba) 6671{ 6672 int tag, ret = 0; 6673 6674 blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret); 6675 if (ret) 6676 goto out; 6677 6678 /* Clear pending task management requests */ 6679 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) { 6680 ret = ufshcd_clear_tm_cmd(hba, tag); 6681 if (ret) 6682 goto out; 6683 } 6684 6685out: 6686 /* Complete the requests that are cleared by s/w */ 6687 ufshcd_complete_requests(hba, false); 6688 6689 return ret != 0; 6690} 6691 6692/** 6693 * ufshcd_err_handler - handle UFS errors that require s/w attention 6694 * @work: pointer to work structure 6695 */ 6696static void ufshcd_err_handler(struct work_struct *work) 6697{ 6698 int retries = MAX_ERR_HANDLER_RETRIES; 6699 struct ufs_hba *hba; 6700 unsigned long flags; 6701 bool needs_restore; 6702 bool needs_reset; 6703 int pmc_err; 6704 6705 hba = container_of(work, struct ufs_hba, eh_work); 6706 6707 dev_info(hba->dev, 6708 "%s started; HBA state %s; powered %d; shutting down %d; saved_err = 0x%x; saved_uic_err = 0x%x; force_reset = %d%s\n", 6709 __func__, ufshcd_state_name[hba->ufshcd_state], 6710 hba->is_powered, hba->shutting_down, hba->saved_err, 6711 hba->saved_uic_err, hba->force_reset, 6712 ufshcd_is_link_broken(hba) ? "; link is broken" : ""); 6713 6714 if (hba->ufs_device_wlun) { 6715 /* 6716 * Use ufshcd_rpm_get_noresume() here to safely perform link 6717 * recovery even if an error occurs during runtime suspend or 6718 * runtime resume. This avoids potential deadlocks that could 6719 * happen if we tried to resume the device while a PM operation 6720 * is already in progress. 6721 */ 6722 ufshcd_rpm_get_noresume(hba); 6723 if (hba->pm_op_in_progress) { 6724 ufshcd_link_recovery(hba); 6725 ufshcd_rpm_put(hba); 6726 return; 6727 } 6728 ufshcd_rpm_put(hba); 6729 } 6730 6731 down(&hba->host_sem); 6732 spin_lock_irqsave(hba->host->host_lock, flags); 6733 if (ufshcd_err_handling_should_stop(hba)) { 6734 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) 6735 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 6736 spin_unlock_irqrestore(hba->host->host_lock, flags); 6737 up(&hba->host_sem); 6738 return; 6739 } 6740 spin_unlock_irqrestore(hba->host->host_lock, flags); 6741 6742 ufshcd_err_handling_prepare(hba); 6743 6744 spin_lock_irqsave(hba->host->host_lock, flags); 6745 ufshcd_set_eh_in_progress(hba); 6746 spin_unlock_irqrestore(hba->host->host_lock, flags); 6747 6748 /* Complete requests that have door-bell cleared by h/w */ 6749 ufshcd_complete_requests(hba, false); 6750 spin_lock_irqsave(hba->host->host_lock, flags); 6751again: 6752 needs_restore = false; 6753 needs_reset = false; 6754 6755 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) 6756 hba->ufshcd_state = UFSHCD_STATE_RESET; 6757 /* 6758 * A full reset and restore might have happened after preparation 6759 * is finished, double check whether we should stop. 6760 */ 6761 if (ufshcd_err_handling_should_stop(hba)) 6762 goto skip_err_handling; 6763 6764 if ((hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) && 6765 !hba->force_reset) { 6766 bool ret; 6767 6768 spin_unlock_irqrestore(hba->host->host_lock, flags); 6769 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */ 6770 ret = ufshcd_quirk_dl_nac_errors(hba); 6771 spin_lock_irqsave(hba->host->host_lock, flags); 6772 if (!ret && ufshcd_err_handling_should_stop(hba)) 6773 goto skip_err_handling; 6774 } 6775 6776 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || 6777 (hba->saved_uic_err && 6778 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { 6779 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR); 6780 6781 spin_unlock_irqrestore(hba->host->host_lock, flags); 6782 ufshcd_print_host_state(hba); 6783 ufshcd_print_pwr_info(hba); 6784 ufshcd_print_evt_hist(hba); 6785 ufshcd_print_tmrs(hba, hba->outstanding_tasks); 6786 ufshcd_print_trs_all(hba, pr_prdt); 6787 spin_lock_irqsave(hba->host->host_lock, flags); 6788 } 6789 6790 /* 6791 * if host reset is required then skip clearing the pending 6792 * transfers forcefully because they will get cleared during 6793 * host reset and restore 6794 */ 6795 if (hba->force_reset || ufshcd_is_link_broken(hba) || 6796 ufshcd_is_saved_err_fatal(hba) || 6797 ((hba->saved_err & UIC_ERROR) && 6798 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR | 6799 UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) { 6800 needs_reset = true; 6801 goto do_reset; 6802 } 6803 6804 /* 6805 * If LINERESET was caught, UFS might have been put to PWM mode, 6806 * check if power mode restore is needed. 6807 */ 6808 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) { 6809 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR; 6810 if (!hba->saved_uic_err) 6811 hba->saved_err &= ~UIC_ERROR; 6812 spin_unlock_irqrestore(hba->host->host_lock, flags); 6813 if (ufshcd_is_pwr_mode_restore_needed(hba)) 6814 needs_restore = true; 6815 spin_lock_irqsave(hba->host->host_lock, flags); 6816 if (!hba->saved_err && !needs_restore) 6817 goto skip_err_handling; 6818 } 6819 6820 hba->silence_err_logs = true; 6821 /* release lock as clear command might sleep */ 6822 spin_unlock_irqrestore(hba->host->host_lock, flags); 6823 6824 needs_reset = ufshcd_abort_all(hba); 6825 6826 spin_lock_irqsave(hba->host->host_lock, flags); 6827 hba->silence_err_logs = false; 6828 if (needs_reset) 6829 goto do_reset; 6830 6831 /* 6832 * After all reqs and tasks are cleared from doorbell, 6833 * now it is safe to retore power mode. 6834 */ 6835 if (needs_restore) { 6836 spin_unlock_irqrestore(hba->host->host_lock, flags); 6837 /* 6838 * Hold the scaling lock just in case dev cmds 6839 * are sent via bsg and/or sysfs. 6840 */ 6841 down_write(&hba->clk_scaling_lock); 6842 hba->force_pmc = true; 6843 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info)); 6844 if (pmc_err) { 6845 needs_reset = true; 6846 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n", 6847 __func__, pmc_err); 6848 } 6849 hba->force_pmc = false; 6850 ufshcd_print_pwr_info(hba); 6851 up_write(&hba->clk_scaling_lock); 6852 spin_lock_irqsave(hba->host->host_lock, flags); 6853 } 6854 6855do_reset: 6856 /* Fatal errors need reset */ 6857 if (needs_reset) { 6858 int err; 6859 6860 hba->force_reset = false; 6861 spin_unlock_irqrestore(hba->host->host_lock, flags); 6862 err = ufshcd_reset_and_restore(hba); 6863 if (err) 6864 dev_err(hba->dev, "%s: reset and restore failed with err %d\n", 6865 __func__, err); 6866 else 6867 ufshcd_recover_pm_error(hba); 6868 spin_lock_irqsave(hba->host->host_lock, flags); 6869 } 6870 6871skip_err_handling: 6872 if (!needs_reset) { 6873 if (hba->ufshcd_state == UFSHCD_STATE_RESET) 6874 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 6875 if (hba->saved_err || hba->saved_uic_err) 6876 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x", 6877 __func__, hba->saved_err, hba->saved_uic_err); 6878 } 6879 /* Exit in an operational state or dead */ 6880 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL && 6881 hba->ufshcd_state != UFSHCD_STATE_ERROR) { 6882 if (--retries) 6883 goto again; 6884 hba->ufshcd_state = UFSHCD_STATE_ERROR; 6885 } 6886 ufshcd_clear_eh_in_progress(hba); 6887 spin_unlock_irqrestore(hba->host->host_lock, flags); 6888 ufshcd_err_handling_unprepare(hba); 6889 up(&hba->host_sem); 6890 6891 dev_info(hba->dev, "%s finished; HBA state %s\n", __func__, 6892 ufshcd_state_name[hba->ufshcd_state]); 6893} 6894 6895/** 6896 * ufshcd_update_uic_error - check and set fatal UIC error flags. 6897 * @hba: per-adapter instance 6898 * 6899 * Return: 6900 * IRQ_HANDLED - If interrupt is valid 6901 * IRQ_NONE - If invalid interrupt 6902 */ 6903static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) 6904{ 6905 u32 reg; 6906 irqreturn_t retval = IRQ_NONE; 6907 6908 /* PHY layer error */ 6909 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); 6910 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) && 6911 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) { 6912 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg); 6913 /* 6914 * To know whether this error is fatal or not, DB timeout 6915 * must be checked but this error is handled separately. 6916 */ 6917 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK) 6918 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", 6919 __func__); 6920 6921 /* Got a LINERESET indication. */ 6922 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) { 6923 struct uic_command *cmd = NULL; 6924 6925 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR; 6926 if (hba->uic_async_done && hba->active_uic_cmd) 6927 cmd = hba->active_uic_cmd; 6928 /* 6929 * Ignore the LINERESET during power mode change 6930 * operation via DME_SET command. 6931 */ 6932 if (cmd && (cmd->command == UIC_CMD_DME_SET)) 6933 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR; 6934 } 6935 retval |= IRQ_HANDLED; 6936 } 6937 6938 /* PA_INIT_ERROR is fatal and needs UIC reset */ 6939 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); 6940 if ((reg & UIC_DATA_LINK_LAYER_ERROR) && 6941 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) { 6942 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg); 6943 6944 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) 6945 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR; 6946 else if (hba->dev_quirks & 6947 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) { 6948 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED) 6949 hba->uic_error |= 6950 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR; 6951 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT) 6952 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR; 6953 } 6954 retval |= IRQ_HANDLED; 6955 } 6956 6957 /* UIC NL/TL/DME errors needs software retry */ 6958 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER); 6959 if ((reg & UIC_NETWORK_LAYER_ERROR) && 6960 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) { 6961 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg); 6962 hba->uic_error |= UFSHCD_UIC_NL_ERROR; 6963 retval |= IRQ_HANDLED; 6964 } 6965 6966 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER); 6967 if ((reg & UIC_TRANSPORT_LAYER_ERROR) && 6968 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) { 6969 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg); 6970 hba->uic_error |= UFSHCD_UIC_TL_ERROR; 6971 retval |= IRQ_HANDLED; 6972 } 6973 6974 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME); 6975 if ((reg & UIC_DME_ERROR) && 6976 (reg & UIC_DME_ERROR_CODE_MASK)) { 6977 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg); 6978 hba->uic_error |= UFSHCD_UIC_DME_ERROR; 6979 retval |= IRQ_HANDLED; 6980 } 6981 6982 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n", 6983 __func__, hba->uic_error); 6984 return retval; 6985} 6986 6987/** 6988 * ufshcd_check_errors - Check for errors that need s/w attention 6989 * @hba: per-adapter instance 6990 * @intr_status: interrupt status generated by the controller 6991 * 6992 * Return: 6993 * IRQ_HANDLED - If interrupt is valid 6994 * IRQ_NONE - If invalid interrupt 6995 */ 6996static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) 6997{ 6998 bool queue_eh_work = false; 6999 irqreturn_t retval = IRQ_NONE; 7000 7001 guard(spinlock_irqsave)(hba->host->host_lock); 7002 hba->errors |= UFSHCD_ERROR_MASK & intr_status; 7003 7004 if (hba->errors & INT_FATAL_ERRORS) { 7005 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR, 7006 hba->errors); 7007 queue_eh_work = true; 7008 } 7009 7010 if (hba->errors & UIC_ERROR) { 7011 hba->uic_error = 0; 7012 retval = ufshcd_update_uic_error(hba); 7013 if (hba->uic_error) 7014 queue_eh_work = true; 7015 } 7016 7017 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) { 7018 dev_err(hba->dev, 7019 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n", 7020 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ? 7021 "Enter" : "Exit", 7022 hba->errors, ufshcd_get_upmcrs(hba)); 7023 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR, 7024 hba->errors); 7025 ufshcd_set_link_broken(hba); 7026 queue_eh_work = true; 7027 } 7028 7029 if (queue_eh_work) { 7030 /* 7031 * update the transfer error masks to sticky bits, let's do this 7032 * irrespective of current ufshcd_state. 7033 */ 7034 hba->saved_err |= hba->errors; 7035 hba->saved_uic_err |= hba->uic_error; 7036 7037 /* dump controller state before resetting */ 7038 if ((hba->saved_err & 7039 (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || 7040 (hba->saved_uic_err && 7041 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { 7042 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n", 7043 __func__, hba->saved_err, 7044 hba->saved_uic_err); 7045 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, 7046 "host_regs: "); 7047 ufshcd_print_pwr_info(hba); 7048 } 7049 ufshcd_schedule_eh_work(hba); 7050 retval |= IRQ_HANDLED; 7051 } 7052 /* 7053 * if (!queue_eh_work) - 7054 * Other errors are either non-fatal where host recovers 7055 * itself without s/w intervention or errors that will be 7056 * handled by the SCSI core layer. 7057 */ 7058 hba->errors = 0; 7059 hba->uic_error = 0; 7060 7061 return retval; 7062} 7063 7064/** 7065 * ufshcd_tmc_handler - handle task management function completion 7066 * @hba: per adapter instance 7067 * 7068 * Return: 7069 * IRQ_HANDLED - If interrupt is valid 7070 * IRQ_NONE - If invalid interrupt 7071 */ 7072static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba) 7073{ 7074 unsigned long flags, pending, issued; 7075 irqreturn_t ret = IRQ_NONE; 7076 int tag; 7077 7078 spin_lock_irqsave(hba->host->host_lock, flags); 7079 pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 7080 issued = hba->outstanding_tasks & ~pending; 7081 for_each_set_bit(tag, &issued, hba->nutmrs) { 7082 struct request *req = hba->tmf_rqs[tag]; 7083 struct completion *c = req->end_io_data; 7084 7085 complete(c); 7086 ret = IRQ_HANDLED; 7087 } 7088 spin_unlock_irqrestore(hba->host->host_lock, flags); 7089 7090 return ret; 7091} 7092 7093/** 7094 * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events 7095 * @hba: per adapter instance 7096 * 7097 * Return: IRQ_HANDLED if interrupt is handled. 7098 */ 7099static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) 7100{ 7101 struct ufs_hw_queue *hwq; 7102 unsigned long outstanding_cqs; 7103 unsigned int nr_queues; 7104 int i, ret; 7105 u32 events; 7106 7107 ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs); 7108 if (ret) 7109 outstanding_cqs = (1ULL << hba->nr_hw_queues) - 1; 7110 7111 /* Exclude the poll queues */ 7112 nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; 7113 for_each_set_bit(i, &outstanding_cqs, nr_queues) { 7114 hwq = &hba->uhq[i]; 7115 7116 events = ufshcd_mcq_read_cqis(hba, i); 7117 if (events) 7118 ufshcd_mcq_write_cqis(hba, events, i); 7119 7120 if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS) 7121 ufshcd_mcq_poll_cqe_lock(hba, hwq); 7122 } 7123 7124 return IRQ_HANDLED; 7125} 7126 7127/** 7128 * ufshcd_sl_intr - Interrupt service routine 7129 * @hba: per adapter instance 7130 * @intr_status: contains interrupts generated by the controller 7131 * 7132 * Return: 7133 * IRQ_HANDLED - If interrupt is valid 7134 * IRQ_NONE - If invalid interrupt 7135 */ 7136static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) 7137{ 7138 irqreturn_t retval = IRQ_NONE; 7139 7140 if (intr_status & UFSHCD_UIC_MASK) 7141 retval |= ufshcd_uic_cmd_compl(hba, intr_status); 7142 7143 if (intr_status & UFSHCD_ERROR_MASK || hba->errors) 7144 retval |= ufshcd_check_errors(hba, intr_status); 7145 7146 if (intr_status & UTP_TASK_REQ_COMPL) 7147 retval |= ufshcd_tmc_handler(hba); 7148 7149 if (intr_status & UTP_TRANSFER_REQ_COMPL) 7150 retval |= ufshcd_transfer_req_compl(hba); 7151 7152 if (intr_status & MCQ_CQ_EVENT_STATUS) 7153 retval |= ufshcd_handle_mcq_cq_events(hba); 7154 7155 return retval; 7156} 7157 7158/** 7159 * ufshcd_threaded_intr - Threaded interrupt service routine 7160 * @irq: irq number 7161 * @__hba: pointer to adapter instance 7162 * 7163 * Return: 7164 * IRQ_HANDLED - If interrupt is valid 7165 * IRQ_NONE - If invalid interrupt 7166 */ 7167static irqreturn_t ufshcd_threaded_intr(int irq, void *__hba) 7168{ 7169 u32 last_intr_status, intr_status, enabled_intr_status = 0; 7170 irqreturn_t retval = IRQ_NONE; 7171 struct ufs_hba *hba = __hba; 7172 int retries = hba->nutrs; 7173 7174 last_intr_status = intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7175 7176 /* 7177 * There could be max of hba->nutrs reqs in flight and in worst case 7178 * if the reqs get finished 1 by 1 after the interrupt status is 7179 * read, make sure we handle them by checking the interrupt status 7180 * again in a loop until we process all of the reqs before returning. 7181 */ 7182 while (intr_status && retries--) { 7183 enabled_intr_status = 7184 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 7185 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 7186 if (enabled_intr_status) 7187 retval |= ufshcd_sl_intr(hba, enabled_intr_status); 7188 7189 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7190 } 7191 7192 if (enabled_intr_status && retval == IRQ_NONE && 7193 (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) || 7194 hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) { 7195 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n", 7196 __func__, 7197 intr_status, 7198 last_intr_status, 7199 enabled_intr_status); 7200 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); 7201 } 7202 7203 return retval; 7204} 7205 7206/** 7207 * ufshcd_intr - Main interrupt service routine 7208 * @irq: irq number 7209 * @__hba: pointer to adapter instance 7210 * 7211 * Return: 7212 * IRQ_HANDLED - If interrupt is valid 7213 * IRQ_WAKE_THREAD - If handling is moved to threaded handled 7214 * IRQ_NONE - If invalid interrupt 7215 */ 7216static irqreturn_t ufshcd_intr(int irq, void *__hba) 7217{ 7218 struct ufs_hba *hba = __hba; 7219 u32 intr_status, enabled_intr_status; 7220 7221 /* Move interrupt handling to thread when MCQ & ESI are not enabled */ 7222 if (!hba->mcq_enabled || !hba->mcq_esi_enabled) 7223 return IRQ_WAKE_THREAD; 7224 7225 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 7226 enabled_intr_status = intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 7227 7228 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 7229 7230 /* Directly handle interrupts since MCQ ESI handlers does the hard job */ 7231 return ufshcd_sl_intr(hba, enabled_intr_status); 7232} 7233 7234static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) 7235{ 7236 int err = 0; 7237 u32 mask = 1 << tag; 7238 7239 if (!test_bit(tag, &hba->outstanding_tasks)) 7240 goto out; 7241 7242 ufshcd_utmrl_clear(hba, tag); 7243 7244 /* poll for max. 1 sec to clear door bell register by h/w */ 7245 err = ufshcd_wait_for_register(hba, 7246 REG_UTP_TASK_REQ_DOOR_BELL, 7247 mask, 0, 1000, 1000); 7248 7249 dev_err(hba->dev, "Clearing task management function with tag %d %s\n", 7250 tag, err < 0 ? "failed" : "succeeded"); 7251 7252out: 7253 return err; 7254} 7255 7256static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, 7257 struct utp_task_req_desc *treq, u8 tm_function) 7258{ 7259 struct request_queue *q = hba->tmf_queue; 7260 struct Scsi_Host *host = hba->host; 7261 DECLARE_COMPLETION_ONSTACK(wait); 7262 struct request *req; 7263 unsigned long flags; 7264 int task_tag, err; 7265 7266 /* 7267 * blk_mq_alloc_request() is used here only to get a free tag. 7268 */ 7269 req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0); 7270 if (IS_ERR(req)) 7271 return PTR_ERR(req); 7272 7273 req->end_io_data = &wait; 7274 ufshcd_hold(hba); 7275 7276 spin_lock_irqsave(host->host_lock, flags); 7277 7278 task_tag = req->tag; 7279 hba->tmf_rqs[req->tag] = req; 7280 treq->upiu_req.req_header.task_tag = task_tag; 7281 7282 memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq)); 7283 ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function); 7284 7285 __set_bit(task_tag, &hba->outstanding_tasks); 7286 7287 spin_unlock_irqrestore(host->host_lock, flags); 7288 7289 /* send command to the controller */ 7290 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL); 7291 7292 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND); 7293 7294 /* wait until the task management command is completed */ 7295 err = wait_for_completion_io_timeout(&wait, 7296 msecs_to_jiffies(TM_CMD_TIMEOUT)); 7297 if (!err) { 7298 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR); 7299 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n", 7300 __func__, tm_function); 7301 if (ufshcd_clear_tm_cmd(hba, task_tag)) 7302 dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n", 7303 __func__, task_tag); 7304 err = -ETIMEDOUT; 7305 } else { 7306 err = 0; 7307 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq)); 7308 7309 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP); 7310 } 7311 7312 spin_lock_irqsave(hba->host->host_lock, flags); 7313 hba->tmf_rqs[req->tag] = NULL; 7314 __clear_bit(task_tag, &hba->outstanding_tasks); 7315 spin_unlock_irqrestore(hba->host->host_lock, flags); 7316 7317 ufshcd_release(hba); 7318 blk_mq_free_request(req); 7319 7320 return err; 7321} 7322 7323/** 7324 * ufshcd_issue_tm_cmd - issues task management commands to controller 7325 * @hba: per adapter instance 7326 * @lun_id: LUN ID to which TM command is sent 7327 * @task_id: task ID to which the TM command is applicable 7328 * @tm_function: task management function opcode 7329 * @tm_response: task management service response return value 7330 * 7331 * Return: non-zero value on error, zero on success. 7332 */ 7333static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, 7334 u8 tm_function, u8 *tm_response) 7335{ 7336 struct utp_task_req_desc treq = { }; 7337 enum utp_ocs ocs_value; 7338 int err; 7339 7340 /* Configure task request descriptor */ 7341 treq.header.interrupt = 1; 7342 treq.header.ocs = OCS_INVALID_COMMAND_STATUS; 7343 7344 /* Configure task request UPIU */ 7345 treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ; 7346 treq.upiu_req.req_header.lun = lun_id; 7347 treq.upiu_req.req_header.tm_function = tm_function; 7348 7349 /* 7350 * The host shall provide the same value for LUN field in the basic 7351 * header and for Input Parameter. 7352 */ 7353 treq.upiu_req.input_param1 = cpu_to_be32(lun_id); 7354 treq.upiu_req.input_param2 = cpu_to_be32(task_id); 7355 7356 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function); 7357 if (err == -ETIMEDOUT) 7358 return err; 7359 7360 ocs_value = treq.header.ocs & MASK_OCS; 7361 if (ocs_value != OCS_SUCCESS) 7362 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", 7363 __func__, ocs_value); 7364 else if (tm_response) 7365 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) & 7366 MASK_TM_SERVICE_RESP; 7367 return err; 7368} 7369 7370/** 7371 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests 7372 * @hba: per-adapter instance 7373 * @req_upiu: upiu request 7374 * @rsp_upiu: upiu reply 7375 * @desc_buff: pointer to descriptor buffer, NULL if NA 7376 * @buff_len: descriptor size, 0 if NA 7377 * @cmd_type: specifies the type (NOP, Query...) 7378 * @desc_op: descriptor operation 7379 * 7380 * Those type of requests uses UTP Transfer Request Descriptor - utrd. 7381 * Therefore, it "rides" the device management infrastructure: uses its tag and 7382 * tasks work queues. 7383 * 7384 * Since there is only one available tag for device management commands, 7385 * the caller is expected to hold the hba->dev_cmd.lock mutex. 7386 * 7387 * Return: 0 upon success; < 0 upon failure. 7388 */ 7389static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, 7390 struct utp_upiu_req *req_upiu, 7391 struct utp_upiu_req *rsp_upiu, 7392 u8 *desc_buff, int *buff_len, 7393 enum dev_cmd_type cmd_type, 7394 enum query_opcode desc_op) 7395{ 7396 struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); 7397 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7398 u32 tag; 7399 int err = 0; 7400 u8 upiu_flags; 7401 7402 /* Protects use of hba->dev_cmd. */ 7403 lockdep_assert_held(&hba->dev_cmd.lock); 7404 7405 if (WARN_ON_ONCE(!cmd)) 7406 return -ENOMEM; 7407 7408 tag = scsi_cmd_to_rq(cmd)->tag; 7409 7410 ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); 7411 7412 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); 7413 7414 /* update the task tag in the request upiu */ 7415 req_upiu->header.task_tag = tag; 7416 7417 /* just copy the upiu request as it is */ 7418 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); 7419 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) { 7420 /* The Data Segment Area is optional depending upon the query 7421 * function value. for WRITE DESCRIPTOR, the data segment 7422 * follows right after the tsf. 7423 */ 7424 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len); 7425 *buff_len = 0; 7426 } 7427 7428 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7429 7430 err = ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); 7431 if (err) 7432 goto put_dev_mgmt_cmd; 7433 7434 /* just copy the upiu response as it is */ 7435 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); 7436 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) { 7437 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu); 7438 u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header 7439 .data_segment_length); 7440 7441 if (*buff_len >= resp_len) { 7442 memcpy(desc_buff, descp, resp_len); 7443 *buff_len = resp_len; 7444 } else { 7445 dev_warn(hba->dev, 7446 "%s: rsp size %d is bigger than buffer size %d", 7447 __func__, resp_len, *buff_len); 7448 *buff_len = 0; 7449 err = -EINVAL; 7450 } 7451 } 7452 7453put_dev_mgmt_cmd: 7454 ufshcd_put_dev_mgmt_cmd(cmd); 7455 7456 return err; 7457} 7458 7459/** 7460 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands 7461 * @hba: per-adapter instance 7462 * @req_upiu: upiu request 7463 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands 7464 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target 7465 * @desc_buff: pointer to descriptor buffer, NULL if NA 7466 * @buff_len: descriptor size, 0 if NA 7467 * @desc_op: descriptor operation 7468 * 7469 * Supports UTP Transfer requests (nop and query), and UTP Task 7470 * Management requests. 7471 * It is up to the caller to fill the upiu conent properly, as it will 7472 * be copied without any further input validations. 7473 * 7474 * Return: 0 upon success; < 0 upon failure. 7475 */ 7476int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, 7477 struct utp_upiu_req *req_upiu, 7478 struct utp_upiu_req *rsp_upiu, 7479 enum upiu_request_transaction msgcode, 7480 u8 *desc_buff, int *buff_len, 7481 enum query_opcode desc_op) 7482{ 7483 int err; 7484 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY; 7485 struct utp_task_req_desc treq = { }; 7486 enum utp_ocs ocs_value; 7487 u8 tm_f = req_upiu->header.tm_function; 7488 7489 switch (msgcode) { 7490 case UPIU_TRANSACTION_NOP_OUT: 7491 cmd_type = DEV_CMD_TYPE_NOP; 7492 fallthrough; 7493 case UPIU_TRANSACTION_QUERY_REQ: 7494 ufshcd_dev_man_lock(hba); 7495 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu, 7496 desc_buff, buff_len, 7497 cmd_type, desc_op); 7498 ufshcd_dev_man_unlock(hba); 7499 7500 break; 7501 case UPIU_TRANSACTION_TASK_REQ: 7502 treq.header.interrupt = 1; 7503 treq.header.ocs = OCS_INVALID_COMMAND_STATUS; 7504 7505 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu)); 7506 7507 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f); 7508 if (err == -ETIMEDOUT) 7509 break; 7510 7511 ocs_value = treq.header.ocs & MASK_OCS; 7512 if (ocs_value != OCS_SUCCESS) { 7513 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, 7514 ocs_value); 7515 break; 7516 } 7517 7518 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu)); 7519 7520 break; 7521 default: 7522 err = -EINVAL; 7523 7524 break; 7525 } 7526 7527 return err; 7528} 7529 7530/** 7531 * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request 7532 * @hba: per adapter instance 7533 * @req_upiu: upiu request 7534 * @rsp_upiu: upiu reply 7535 * @req_ehs: EHS field which contains Advanced RPMB Request Message 7536 * @rsp_ehs: EHS field which returns Advanced RPMB Response Message 7537 * @sg_cnt: The number of sg lists actually used 7538 * @sg_list: Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation 7539 * @dir: DMA direction 7540 * 7541 * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; 7542 * < 0 if another error occurred. 7543 */ 7544int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, 7545 struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs, 7546 struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list, 7547 enum dma_data_direction dir) 7548{ 7549 struct scsi_cmnd *cmd; 7550 struct ufshcd_lrb *lrbp; 7551 u32 tag; 7552 int err = 0; 7553 int result; 7554 u8 upiu_flags; 7555 u8 *ehs_data; 7556 u16 ehs_len; 7557 int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0; 7558 7559 ufshcd_dev_man_lock(hba); 7560 7561 cmd = ufshcd_get_dev_mgmt_cmd(hba); 7562 7563 if (WARN_ON_ONCE(!cmd)) { 7564 err = -ENOMEM; 7565 goto unlock; 7566 } 7567 7568 lrbp = scsi_cmd_priv(cmd); 7569 tag = scsi_cmd_to_rq(cmd)->tag; 7570 7571 ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, 7572 tag); 7573 7574 ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs); 7575 7576 /* update the task tag */ 7577 req_upiu->header.task_tag = tag; 7578 7579 /* copy the UPIU(contains CDB) request as it is */ 7580 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); 7581 /* Copy EHS, starting with byte32, immediately after the CDB package */ 7582 memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs)); 7583 7584 if (dir != DMA_NONE && sg_list) 7585 ufshcd_sgl_to_prdt(hba, lrbp, sg_cnt, sg_list); 7586 7587 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7588 7589 err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT); 7590 if (err) 7591 goto put_dev_mgmt_cmd; 7592 7593 err = ufshcd_dev_cmd_completion(hba, lrbp); 7594 if (!err) { 7595 /* Just copy the upiu response as it is */ 7596 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); 7597 /* Get the response UPIU result */ 7598 result = (lrbp->ucd_rsp_ptr->header.response << 8) | 7599 lrbp->ucd_rsp_ptr->header.status; 7600 7601 ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length; 7602 /* 7603 * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data 7604 * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB 7605 * Message is 02h 7606 */ 7607 if (ehs_len == 2 && rsp_ehs) { 7608 /* 7609 * ucd_rsp_ptr points to a buffer with a length of 512 bytes 7610 * (ALIGNED_UPIU_SIZE = 512), and the EHS data just starts from byte32 7611 */ 7612 ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE; 7613 memcpy(rsp_ehs, ehs_data, ehs_len * 32); 7614 } 7615 } 7616 7617put_dev_mgmt_cmd: 7618 ufshcd_put_dev_mgmt_cmd(cmd); 7619 7620unlock: 7621 ufshcd_dev_man_unlock(hba); 7622 7623 return err ? : result; 7624} 7625 7626static bool ufshcd_clear_lu_cmds(struct request *req, void *priv) 7627{ 7628 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); 7629 struct scsi_device *sdev = cmd->device; 7630 struct Scsi_Host *shost = sdev->host; 7631 struct ufs_hba *hba = shost_priv(shost); 7632 const u64 lun = *(u64 *)priv; 7633 const u32 tag = req->tag; 7634 7635 if (blk_mq_is_reserved_rq(req) || sdev->lun != lun) 7636 return true; 7637 7638 if (ufshcd_clear_cmd(hba, tag) < 0) { 7639 dev_err(hba->dev, "%s: failed to clear request %d\n", __func__, 7640 tag); 7641 return true; 7642 } 7643 7644 if (hba->mcq_enabled) { 7645 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, req); 7646 7647 if (hwq) 7648 ufshcd_mcq_poll_cqe_lock(hba, hwq); 7649 return true; 7650 } 7651 7652 ufshcd_compl_one_cqe(hba, tag, NULL); 7653 return true; 7654} 7655 7656/** 7657 * ufshcd_eh_device_reset_handler() - Reset a single logical unit. 7658 * @cmd: SCSI command pointer 7659 * 7660 * Return: SUCCESS or FAILED. 7661 */ 7662static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) 7663{ 7664 struct Scsi_Host *host; 7665 struct ufs_hba *hba; 7666 int err; 7667 u8 resp = 0xF, lun; 7668 7669 host = cmd->device->host; 7670 hba = shost_priv(host); 7671 7672 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); 7673 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp); 7674 if (err) { 7675 } else if (resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7676 err = resp; 7677 } else { 7678 /* clear the commands that were pending for corresponding LUN */ 7679 blk_mq_tagset_busy_iter(&hba->host->tag_set, 7680 ufshcd_clear_lu_cmds, 7681 &cmd->device->lun); 7682 } 7683 7684 hba->req_abort_count = 0; 7685 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err); 7686 if (!err) { 7687 err = SUCCESS; 7688 } else { 7689 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7690 err = FAILED; 7691 } 7692 return err; 7693} 7694 7695static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) 7696{ 7697 int tag; 7698 7699 for_each_set_bit(tag, &bitmap, hba->nutrs) { 7700 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7701 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7702 7703 lrbp->req_abort_skip = true; 7704 } 7705} 7706 7707/** 7708 * ufshcd_try_to_abort_task - abort a specific task 7709 * @hba: Pointer to adapter instance 7710 * @tag: Tag of the task to be aborted 7711 * 7712 * Abort the pending command in device by sending UFS_ABORT_TASK task management 7713 * command, and in host controller by clearing the door-bell register. There can 7714 * be race between controller sending the command to the device while abort is 7715 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is 7716 * really issued and then try to abort it. 7717 * 7718 * Return: zero on success, non-zero on failure. 7719 */ 7720int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) 7721{ 7722 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); 7723 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7724 int err; 7725 int poll_cnt; 7726 u8 resp = 0xF; 7727 7728 for (poll_cnt = 100; poll_cnt; poll_cnt--) { 7729 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_QUERY_TASK, 7730 &resp); 7731 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { 7732 /* cmd pending in the device */ 7733 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n", 7734 __func__, tag); 7735 break; 7736 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7737 /* 7738 * cmd not pending in the device, check if it is 7739 * in transition. 7740 */ 7741 dev_info( 7742 hba->dev, 7743 "%s: cmd with tag %d not pending in the device.\n", 7744 __func__, tag); 7745 if (!ufshcd_cmd_inflight(cmd)) { 7746 dev_info(hba->dev, 7747 "%s: cmd with tag=%d completed.\n", 7748 __func__, tag); 7749 return 0; 7750 } 7751 usleep_range(100, 200); 7752 } else { 7753 dev_err(hba->dev, 7754 "%s: no response from device. tag = %d, err %d\n", 7755 __func__, tag, err); 7756 return err ? : resp; 7757 } 7758 } 7759 7760 if (!poll_cnt) 7761 return -EBUSY; 7762 7763 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_ABORT_TASK, &resp); 7764 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7765 if (!err) { 7766 err = resp; /* service response error */ 7767 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n", 7768 __func__, tag, err); 7769 } 7770 return err; 7771 } 7772 7773 err = ufshcd_clear_cmd(hba, tag); 7774 if (err) 7775 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n", 7776 __func__, tag, err); 7777 7778 return err; 7779} 7780 7781/** 7782 * ufshcd_abort - scsi host template eh_abort_handler callback 7783 * @cmd: SCSI command pointer 7784 * 7785 * Return: SUCCESS or FAILED. 7786 */ 7787static int ufshcd_abort(struct scsi_cmnd *cmd) 7788{ 7789 struct Scsi_Host *host = cmd->device->host; 7790 struct ufs_hba *hba = shost_priv(host); 7791 struct request *rq = scsi_cmd_to_rq(cmd); 7792 int tag = rq->tag; 7793 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 7794 unsigned long flags; 7795 int err = FAILED; 7796 bool outstanding; 7797 u32 reg; 7798 7799 ufshcd_hold(hba); 7800 7801 if (!hba->mcq_enabled) { 7802 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 7803 if (!test_bit(tag, &hba->outstanding_reqs)) { 7804 /* If command is already aborted/completed, return FAILED. */ 7805 dev_err(hba->dev, 7806 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n", 7807 __func__, tag, hba->outstanding_reqs, reg); 7808 goto release; 7809 } 7810 } 7811 7812 /* Print Transfer Request of aborted task */ 7813 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag); 7814 7815 /* 7816 * Print detailed info about aborted request. 7817 * As more than one request might get aborted at the same time, 7818 * print full information only for the first aborted request in order 7819 * to reduce repeated printouts. For other aborted requests only print 7820 * basic details. 7821 */ 7822 if (ufshcd_is_scsi_cmd(cmd)) 7823 scsi_print_command(cmd); 7824 if (!hba->req_abort_count) { 7825 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag); 7826 ufshcd_print_evt_hist(hba); 7827 ufshcd_print_host_state(hba); 7828 ufshcd_print_pwr_info(hba); 7829 ufshcd_print_tr(hba, cmd, true); 7830 } else { 7831 ufshcd_print_tr(hba, cmd, false); 7832 } 7833 hba->req_abort_count++; 7834 7835 if (!hba->mcq_enabled && !(reg & (1 << tag))) { 7836 /* only execute this code in single doorbell mode */ 7837 dev_err(hba->dev, 7838 "%s: cmd was completed, but without a notifying intr, tag = %d", 7839 __func__, tag); 7840 __ufshcd_transfer_req_compl(hba, 1UL << tag); 7841 goto release; 7842 } 7843 7844 /* 7845 * Task abort to the device W-LUN is illegal. When this command 7846 * will fail, due to spec violation, scsi err handling next step 7847 * will be to send LU reset which, again, is a spec violation. 7848 * To avoid these unnecessary/illegal steps, first we clean up 7849 * the lrb taken by this cmd and re-set it in outstanding_reqs, 7850 * then queue the eh_work and bail. 7851 */ 7852 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) { 7853 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun); 7854 7855 spin_lock_irqsave(host->host_lock, flags); 7856 hba->force_reset = true; 7857 ufshcd_schedule_eh_work(hba); 7858 spin_unlock_irqrestore(host->host_lock, flags); 7859 goto release; 7860 } 7861 7862 if (hba->mcq_enabled) { 7863 /* MCQ mode. Branch off to handle abort for mcq mode */ 7864 err = ufshcd_mcq_abort(cmd); 7865 goto release; 7866 } 7867 7868 /* Skip task abort in case previous aborts failed and report failure */ 7869 if (lrbp->req_abort_skip) { 7870 dev_err(hba->dev, "%s: skipping abort\n", __func__); 7871 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); 7872 goto release; 7873 } 7874 7875 if (blk_mq_is_reserved_rq(rq)) 7876 err = ufshcd_clear_cmd(hba, tag); 7877 else 7878 err = ufshcd_try_to_abort_task(hba, tag); 7879 if (err) { 7880 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7881 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); 7882 err = FAILED; 7883 goto release; 7884 } 7885 7886 /* 7887 * Clear the corresponding bit from outstanding_reqs since the command 7888 * has been aborted successfully. 7889 */ 7890 spin_lock_irqsave(&hba->outstanding_lock, flags); 7891 outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs); 7892 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7893 7894 if (outstanding) 7895 ufshcd_release_scsi_cmd(hba, cmd); 7896 7897 err = SUCCESS; 7898 7899release: 7900 /* Matches the ufshcd_hold() call at the start of this function. */ 7901 ufshcd_release(hba); 7902 return err; 7903} 7904 7905/** 7906 * ufshcd_process_probe_result - Process the ufshcd_probe_hba() result. 7907 * @hba: UFS host controller instance. 7908 * @probe_start: time when the ufshcd_probe_hba() call started. 7909 * @ret: ufshcd_probe_hba() return value. 7910 */ 7911static void ufshcd_process_probe_result(struct ufs_hba *hba, 7912 ktime_t probe_start, int ret) 7913{ 7914 unsigned long flags; 7915 7916 spin_lock_irqsave(hba->host->host_lock, flags); 7917 if (ret) 7918 hba->ufshcd_state = UFSHCD_STATE_ERROR; 7919 else if (hba->ufshcd_state == UFSHCD_STATE_RESET) 7920 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 7921 spin_unlock_irqrestore(hba->host->host_lock, flags); 7922 7923 trace_ufshcd_init(hba, ret, 7924 ktime_to_us(ktime_sub(ktime_get(), probe_start)), 7925 hba->curr_dev_pwr_mode, hba->uic_link_state); 7926} 7927 7928/** 7929 * ufshcd_host_reset_and_restore - reset and restore host controller 7930 * @hba: per-adapter instance 7931 * 7932 * Note that host controller reset may issue DME_RESET to 7933 * local and remote (device) Uni-Pro stack and the attributes 7934 * are reset to default state. 7935 * 7936 * Return: zero on success, non-zero on failure. 7937 */ 7938static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) 7939{ 7940 int err; 7941 7942 /* 7943 * Stop the host controller and complete the requests 7944 * cleared by h/w 7945 */ 7946 ufshcd_hba_stop(hba); 7947 hba->silence_err_logs = true; 7948 ufshcd_complete_requests(hba, true); 7949 hba->silence_err_logs = false; 7950 7951 /* scale up clocks to max frequency before full reinitialization */ 7952 if (ufshcd_is_clkscaling_supported(hba)) 7953 ufshcd_scale_clks(hba, ULONG_MAX, true); 7954 7955 err = ufshcd_hba_enable(hba); 7956 7957 /* Establish the link again and restore the device */ 7958 if (!err) { 7959 ktime_t probe_start = ktime_get(); 7960 7961 err = ufshcd_device_init(hba, /*init_dev_params=*/false); 7962 if (!err) 7963 err = ufshcd_probe_hba(hba, false); 7964 ufshcd_process_probe_result(hba, probe_start, err); 7965 } 7966 7967 if (err) 7968 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); 7969 ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err); 7970 return err; 7971} 7972 7973/** 7974 * ufshcd_reset_and_restore - reset and re-initialize host/device 7975 * @hba: per-adapter instance 7976 * 7977 * Reset and recover device, host and re-establish link. This 7978 * is helpful to recover the communication in fatal error conditions. 7979 * 7980 * Return: zero on success, non-zero on failure. 7981 */ 7982static int ufshcd_reset_and_restore(struct ufs_hba *hba) 7983{ 7984 u32 saved_err = 0; 7985 u32 saved_uic_err = 0; 7986 int err = 0; 7987 unsigned long flags; 7988 int retries = MAX_HOST_RESET_RETRIES; 7989 7990 spin_lock_irqsave(hba->host->host_lock, flags); 7991 do { 7992 /* 7993 * This is a fresh start, cache and clear saved error first, 7994 * in case new error generated during reset and restore. 7995 */ 7996 saved_err |= hba->saved_err; 7997 saved_uic_err |= hba->saved_uic_err; 7998 hba->saved_err = 0; 7999 hba->saved_uic_err = 0; 8000 hba->force_reset = false; 8001 hba->ufshcd_state = UFSHCD_STATE_RESET; 8002 spin_unlock_irqrestore(hba->host->host_lock, flags); 8003 8004 /* Reset the attached device */ 8005 ufshcd_device_reset(hba); 8006 8007 err = ufshcd_host_reset_and_restore(hba); 8008 8009 spin_lock_irqsave(hba->host->host_lock, flags); 8010 if (err) 8011 continue; 8012 /* Do not exit unless operational or dead */ 8013 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL && 8014 hba->ufshcd_state != UFSHCD_STATE_ERROR && 8015 hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL) 8016 err = -EAGAIN; 8017 } while (err && --retries); 8018 8019 /* 8020 * Inform scsi mid-layer that we did reset and allow to handle 8021 * Unit Attention properly. 8022 */ 8023 scsi_report_bus_reset(hba->host, 0); 8024 if (err) { 8025 hba->ufshcd_state = UFSHCD_STATE_ERROR; 8026 hba->saved_err |= saved_err; 8027 hba->saved_uic_err |= saved_uic_err; 8028 } 8029 spin_unlock_irqrestore(hba->host->host_lock, flags); 8030 8031 return err; 8032} 8033 8034/** 8035 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer 8036 * @cmd: SCSI command pointer 8037 * 8038 * Return: SUCCESS or FAILED. 8039 */ 8040static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) 8041{ 8042 int err = SUCCESS; 8043 unsigned long flags; 8044 struct ufs_hba *hba; 8045 8046 hba = shost_priv(cmd->device->host); 8047 8048 /* 8049 * If runtime PM sent SSU and got a timeout, scsi_error_handler is 8050 * stuck in this function waiting for flush_work(&hba->eh_work). And 8051 * ufshcd_err_handler(eh_work) is stuck waiting for runtime PM. Do 8052 * ufshcd_link_recovery instead of eh_work to prevent deadlock. 8053 */ 8054 if (hba->pm_op_in_progress) { 8055 if (ufshcd_link_recovery(hba)) 8056 err = FAILED; 8057 8058 return err; 8059 } 8060 8061 spin_lock_irqsave(hba->host->host_lock, flags); 8062 hba->force_reset = true; 8063 ufshcd_schedule_eh_work(hba); 8064 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__); 8065 spin_unlock_irqrestore(hba->host->host_lock, flags); 8066 8067 flush_work(&hba->eh_work); 8068 8069 spin_lock_irqsave(hba->host->host_lock, flags); 8070 if (hba->ufshcd_state == UFSHCD_STATE_ERROR) 8071 err = FAILED; 8072 spin_unlock_irqrestore(hba->host->host_lock, flags); 8073 8074 return err; 8075} 8076 8077/** 8078 * ufshcd_get_max_icc_level - calculate the ICC level 8079 * @sup_curr_uA: max. current supported by the regulator 8080 * @start_scan: row at the desc table to start scan from 8081 * @buff: power descriptor buffer 8082 * 8083 * Return: calculated max ICC level for specific regulator. 8084 */ 8085static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, 8086 const char *buff) 8087{ 8088 int i; 8089 int curr_uA; 8090 u16 data; 8091 u16 unit; 8092 8093 for (i = start_scan; i >= 0; i--) { 8094 data = get_unaligned_be16(&buff[2 * i]); 8095 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >> 8096 ATTR_ICC_LVL_UNIT_OFFSET; 8097 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK; 8098 switch (unit) { 8099 case UFSHCD_NANO_AMP: 8100 curr_uA = curr_uA / 1000; 8101 break; 8102 case UFSHCD_MILI_AMP: 8103 curr_uA = curr_uA * 1000; 8104 break; 8105 case UFSHCD_AMP: 8106 curr_uA = curr_uA * 1000 * 1000; 8107 break; 8108 case UFSHCD_MICRO_AMP: 8109 default: 8110 break; 8111 } 8112 if (sup_curr_uA >= curr_uA) 8113 break; 8114 } 8115 if (i < 0) { 8116 i = 0; 8117 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i); 8118 } 8119 8120 return (u32)i; 8121} 8122 8123/** 8124 * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level 8125 * In case regulators are not initialized we'll return 0 8126 * @hba: per-adapter instance 8127 * @desc_buf: power descriptor buffer to extract ICC levels from. 8128 * 8129 * Return: calculated ICC level. 8130 */ 8131static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba, 8132 const u8 *desc_buf) 8133{ 8134 u32 icc_level = 0; 8135 8136 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq || 8137 !hba->vreg_info.vccq2) { 8138 /* 8139 * Using dev_dbg to avoid messages during runtime PM to avoid 8140 * never-ending cycles of messages written back to storage by 8141 * user space causing runtime resume, causing more messages and 8142 * so on. 8143 */ 8144 dev_dbg(hba->dev, 8145 "%s: Regulator capability was not set, actvIccLevel=%d", 8146 __func__, icc_level); 8147 goto out; 8148 } 8149 8150 if (hba->vreg_info.vcc->max_uA) 8151 icc_level = ufshcd_get_max_icc_level( 8152 hba->vreg_info.vcc->max_uA, 8153 POWER_DESC_MAX_ACTV_ICC_LVLS - 1, 8154 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]); 8155 8156 if (hba->vreg_info.vccq->max_uA) 8157 icc_level = ufshcd_get_max_icc_level( 8158 hba->vreg_info.vccq->max_uA, 8159 icc_level, 8160 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]); 8161 8162 if (hba->vreg_info.vccq2->max_uA) 8163 icc_level = ufshcd_get_max_icc_level( 8164 hba->vreg_info.vccq2->max_uA, 8165 icc_level, 8166 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]); 8167out: 8168 return icc_level; 8169} 8170 8171static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba) 8172{ 8173 int ret; 8174 u8 *desc_buf; 8175 u32 icc_level; 8176 8177 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8178 if (!desc_buf) 8179 return; 8180 8181 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0, 8182 desc_buf, QUERY_DESC_MAX_SIZE); 8183 if (ret) { 8184 dev_err(hba->dev, 8185 "%s: Failed reading power descriptor ret = %d", 8186 __func__, ret); 8187 goto out; 8188 } 8189 8190 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf); 8191 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level); 8192 8193 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8194 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level); 8195 8196 if (ret) 8197 dev_err(hba->dev, 8198 "%s: Failed configuring bActiveICCLevel = %d ret = %d", 8199 __func__, icc_level, ret); 8200 8201out: 8202 kfree(desc_buf); 8203} 8204 8205static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) 8206{ 8207 struct Scsi_Host *shost = sdev->host; 8208 8209 scsi_autopm_get_device(sdev); 8210 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev); 8211 if (sdev->rpm_autosuspend) 8212 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, 8213 shost->rpm_autosuspend_delay); 8214 scsi_autopm_put_device(sdev); 8215} 8216 8217/** 8218 * ufshcd_scsi_add_wlus - Adds required W-LUs 8219 * @hba: per-adapter instance 8220 * 8221 * UFS device specification requires the UFS devices to support 4 well known 8222 * logical units: 8223 * "REPORT_LUNS" (address: 01h) 8224 * "UFS Device" (address: 50h) 8225 * "RPMB" (address: 44h) 8226 * "BOOT" (address: 30h) 8227 * UFS device's power management needs to be controlled by "POWER CONDITION" 8228 * field of SSU (START STOP UNIT) command. But this "power condition" field 8229 * will take effect only when its sent to "UFS device" well known logical unit 8230 * hence we require the scsi_device instance to represent this logical unit in 8231 * order for the UFS host driver to send the SSU command for power management. 8232 * 8233 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory 8234 * Block) LU so user space process can control this LU. User space may also 8235 * want to have access to BOOT LU. 8236 * 8237 * This function adds scsi device instances for each of all well known LUs 8238 * (except "REPORT LUNS" LU). 8239 * 8240 * Return: zero on success (all required W-LUs are added successfully), 8241 * non-zero error value on failure (if failed to add any of the required W-LU). 8242 */ 8243static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) 8244{ 8245 int ret = 0; 8246 struct scsi_device *sdev_boot, *sdev_rpmb; 8247 8248 hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0, 8249 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL); 8250 if (IS_ERR(hba->ufs_device_wlun)) { 8251 ret = PTR_ERR(hba->ufs_device_wlun); 8252 hba->ufs_device_wlun = NULL; 8253 goto out; 8254 } 8255 scsi_device_put(hba->ufs_device_wlun); 8256 8257 sdev_rpmb = __scsi_add_device(hba->host, 0, 0, 8258 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL); 8259 if (IS_ERR(sdev_rpmb)) { 8260 ret = PTR_ERR(sdev_rpmb); 8261 hba->ufs_rpmb_wlun = NULL; 8262 dev_err(hba->dev, "%s: RPMB WLUN not found\n", __func__); 8263 goto remove_ufs_device_wlun; 8264 } 8265 hba->ufs_rpmb_wlun = sdev_rpmb; 8266 ufshcd_blk_pm_runtime_init(sdev_rpmb); 8267 scsi_device_put(sdev_rpmb); 8268 8269 sdev_boot = __scsi_add_device(hba->host, 0, 0, 8270 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL); 8271 if (IS_ERR(sdev_boot)) { 8272 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__); 8273 } else { 8274 ufshcd_blk_pm_runtime_init(sdev_boot); 8275 scsi_device_put(sdev_boot); 8276 } 8277 goto out; 8278 8279remove_ufs_device_wlun: 8280 scsi_remove_device(hba->ufs_device_wlun); 8281out: 8282 return ret; 8283} 8284 8285static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf) 8286{ 8287 struct ufs_dev_info *dev_info = &hba->dev_info; 8288 u8 lun; 8289 u32 d_lu_wb_buf_alloc; 8290 u32 ext_ufs_feature; 8291 8292 if (!ufshcd_is_wb_allowed(hba)) 8293 return; 8294 8295 /* 8296 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or 8297 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES 8298 * enabled 8299 */ 8300 if (!(dev_info->wspecversion >= 0x310 || 8301 dev_info->wspecversion == 0x220 || 8302 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))) 8303 goto wb_disabled; 8304 8305 ext_ufs_feature = get_unaligned_be32(desc_buf + 8306 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8307 8308 if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP)) 8309 goto wb_disabled; 8310 8311 /* 8312 * WB may be supported but not configured while provisioning. The spec 8313 * says, in dedicated wb buffer mode, a max of 1 lun would have wb 8314 * buffer configured. 8315 */ 8316 dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE]; 8317 8318 dev_info->ext_wb_sup = get_unaligned_be16(desc_buf + 8319 DEVICE_DESC_PARAM_EXT_WB_SUP); 8320 8321 dev_info->b_presrv_uspc_en = 8322 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN]; 8323 8324 if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) { 8325 if (!get_unaligned_be32(desc_buf + 8326 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS)) 8327 goto wb_disabled; 8328 } else { 8329 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) { 8330 d_lu_wb_buf_alloc = 0; 8331 ufshcd_read_unit_desc_param(hba, 8332 lun, 8333 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS, 8334 (u8 *)&d_lu_wb_buf_alloc, 8335 sizeof(d_lu_wb_buf_alloc)); 8336 if (d_lu_wb_buf_alloc) { 8337 dev_info->wb_dedicated_lu = lun; 8338 break; 8339 } 8340 } 8341 8342 if (!d_lu_wb_buf_alloc) 8343 goto wb_disabled; 8344 } 8345 8346 if (!ufshcd_is_wb_buf_lifetime_available(hba)) 8347 goto wb_disabled; 8348 8349 return; 8350 8351wb_disabled: 8352 hba->caps &= ~UFSHCD_CAP_WB_EN; 8353} 8354 8355static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf) 8356{ 8357 struct ufs_dev_info *dev_info = &hba->dev_info; 8358 u32 ext_ufs_feature; 8359 u8 mask = 0; 8360 8361 if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300) 8362 return; 8363 8364 ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8365 8366 if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF) 8367 mask |= MASK_EE_TOO_LOW_TEMP; 8368 8369 if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF) 8370 mask |= MASK_EE_TOO_HIGH_TEMP; 8371 8372 if (mask) { 8373 ufshcd_enable_ee(hba, mask); 8374 ufs_hwmon_probe(hba, mask); 8375 } 8376} 8377 8378static void ufshcd_device_lvl_exception_probe(struct ufs_hba *hba, u8 *desc_buf) 8379{ 8380 u32 ext_ufs_feature; 8381 8382 if (hba->dev_info.wspecversion < 0x410) 8383 return; 8384 8385 ext_ufs_feature = get_unaligned_be32(desc_buf + 8386 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 8387 if (!(ext_ufs_feature & UFS_DEV_LVL_EXCEPTION_SUP)) 8388 return; 8389 8390 atomic_set(&hba->dev_lvl_exception_count, 0); 8391 ufshcd_enable_ee(hba, MASK_EE_DEV_LVL_EXCEPTION); 8392} 8393 8394static void ufshcd_set_rtt(struct ufs_hba *hba) 8395{ 8396 struct ufs_dev_info *dev_info = &hba->dev_info; 8397 u32 rtt = 0; 8398 u32 dev_rtt = 0; 8399 int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ? 8400 hba->vops->max_num_rtt : hba->nortt; 8401 8402 /* RTT override makes sense only for UFS-4.0 and above */ 8403 if (dev_info->wspecversion < 0x400) 8404 return; 8405 8406 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 8407 QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &dev_rtt)) { 8408 dev_err(hba->dev, "failed reading bMaxNumOfRTT\n"); 8409 return; 8410 } 8411 8412 /* do not override if it was already written */ 8413 if (dev_rtt != DEFAULT_MAX_NUM_RTT) 8414 return; 8415 8416 rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap); 8417 8418 if (rtt == dev_rtt) 8419 return; 8420 8421 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8422 QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt)) 8423 dev_err(hba->dev, "failed writing bMaxNumOfRTT\n"); 8424} 8425 8426void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, 8427 const struct ufs_dev_quirk *fixups) 8428{ 8429 const struct ufs_dev_quirk *f; 8430 struct ufs_dev_info *dev_info = &hba->dev_info; 8431 8432 if (!fixups) 8433 return; 8434 8435 for (f = fixups; f->quirk; f++) { 8436 if ((f->wmanufacturerid == dev_info->wmanufacturerid || 8437 f->wmanufacturerid == UFS_ANY_VENDOR) && 8438 ((dev_info->model && 8439 STR_PRFX_EQUAL(f->model, dev_info->model)) || 8440 !strcmp(f->model, UFS_ANY_MODEL))) 8441 hba->dev_quirks |= f->quirk; 8442 } 8443} 8444EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks); 8445 8446static void ufs_fixup_device_setup(struct ufs_hba *hba) 8447{ 8448 /* fix by general quirk table */ 8449 ufshcd_fixup_dev_quirks(hba, ufs_fixups); 8450 8451 /* allow vendors to fix quirks */ 8452 ufshcd_vops_fixup_dev_quirks(hba); 8453} 8454 8455static void ufshcd_update_rtc(struct ufs_hba *hba) 8456{ 8457 struct timespec64 ts64; 8458 int err; 8459 u32 val; 8460 8461 ktime_get_real_ts64(&ts64); 8462 8463 if (ts64.tv_sec < hba->dev_info.rtc_time_baseline) { 8464 dev_warn_once(hba->dev, "%s: Current time precedes previous setting!\n", __func__); 8465 return; 8466 } 8467 8468 /* 8469 * The Absolute RTC mode has a 136-year limit, spanning from 2010 to 2146. If a time beyond 8470 * 2146 is required, it is recommended to choose the relative RTC mode. 8471 */ 8472 val = ts64.tv_sec - hba->dev_info.rtc_time_baseline; 8473 8474 /* Skip update RTC if RPM state is not RPM_ACTIVE */ 8475 if (ufshcd_rpm_get_if_active(hba) <= 0) 8476 return; 8477 8478 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_SECONDS_PASSED, 8479 0, 0, &val); 8480 ufshcd_rpm_put(hba); 8481 8482 if (err) 8483 dev_err(hba->dev, "%s: Failed to update rtc %d\n", __func__, err); 8484 else if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) 8485 hba->dev_info.rtc_time_baseline = ts64.tv_sec; 8486} 8487 8488static void ufshcd_rtc_work(struct work_struct *work) 8489{ 8490 struct ufs_hba *hba; 8491 8492 hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work); 8493 8494 /* Update RTC only when there are no requests in progress and UFSHCI is operational */ 8495 if (!ufshcd_is_ufs_dev_busy(hba) && 8496 hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL && 8497 !hba->clk_gating.active_reqs) 8498 ufshcd_update_rtc(hba); 8499 8500 if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period) 8501 schedule_delayed_work(&hba->ufs_rtc_update_work, 8502 msecs_to_jiffies(hba->dev_info.rtc_update_period)); 8503} 8504 8505static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf) 8506{ 8507 u16 periodic_rtc_update = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_FRQ_RTC]); 8508 struct ufs_dev_info *dev_info = &hba->dev_info; 8509 8510 if (periodic_rtc_update & UFS_RTC_TIME_BASELINE) { 8511 dev_info->rtc_type = UFS_RTC_ABSOLUTE; 8512 8513 /* 8514 * The concept of measuring time in Linux as the number of seconds elapsed since 8515 * 00:00:00 UTC on January 1, 1970, and UFS ABS RTC is elapsed from January 1st 8516 * 2010 00:00, here we need to adjust ABS baseline. 8517 */ 8518 dev_info->rtc_time_baseline = mktime64(2010, 1, 1, 0, 0, 0) - 8519 mktime64(1970, 1, 1, 0, 0, 0); 8520 } else { 8521 dev_info->rtc_type = UFS_RTC_RELATIVE; 8522 dev_info->rtc_time_baseline = 0; 8523 } 8524 8525 /* 8526 * We ignore TIME_PERIOD defined in wPeriodicRTCUpdate because Spec does not clearly state 8527 * how to calculate the specific update period for each time unit. And we disable periodic 8528 * RTC update work, let user configure by sysfs node according to specific circumstance. 8529 */ 8530 dev_info->rtc_update_period = 0; 8531} 8532 8533/** 8534 * ufshcd_create_device_id - Generate unique device identifier string 8535 * @hba: per-adapter instance 8536 * @desc_buf: device descriptor buffer 8537 * 8538 * Creates a unique device ID string combining manufacturer ID, spec version, 8539 * model name, serial number (as hex), device version, and manufacture date. 8540 * 8541 * Returns: Allocated device ID string on success, NULL on failure 8542 */ 8543static char *ufshcd_create_device_id(struct ufs_hba *hba, u8 *desc_buf) 8544{ 8545 struct ufs_dev_info *dev_info = &hba->dev_info; 8546 u16 manufacture_date; 8547 u16 device_version; 8548 u8 *serial_number; 8549 char *serial_hex; 8550 char *device_id; 8551 u8 serial_index; 8552 int serial_len; 8553 int ret; 8554 8555 serial_index = desc_buf[DEVICE_DESC_PARAM_SN]; 8556 8557 ret = ufshcd_read_string_desc(hba, serial_index, &serial_number, SD_RAW); 8558 if (ret < 0) { 8559 dev_err(hba->dev, "Failed reading Serial Number. err = %d\n", ret); 8560 return NULL; 8561 } 8562 8563 device_version = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_DEV_VER]); 8564 manufacture_date = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_MANF_DATE]); 8565 8566 serial_len = ret; 8567 /* Allocate buffer for hex string: 2 chars per byte + null terminator */ 8568 serial_hex = kzalloc(serial_len * 2 + 1, GFP_KERNEL); 8569 if (!serial_hex) { 8570 kfree(serial_number); 8571 return NULL; 8572 } 8573 8574 bin2hex(serial_hex, serial_number, serial_len); 8575 8576 /* 8577 * Device ID format is ABI with secure world - do not change without firmware 8578 * coordination. 8579 */ 8580 device_id = kasprintf(GFP_KERNEL, "%04X-%04X-%s-%s-%04X-%04X", 8581 dev_info->wmanufacturerid, dev_info->wspecversion, 8582 dev_info->model, serial_hex, device_version, 8583 manufacture_date); 8584 8585 kfree(serial_hex); 8586 kfree(serial_number); 8587 8588 if (!device_id) 8589 dev_warn(hba->dev, "Failed to allocate unique device ID\n"); 8590 8591 return device_id; 8592} 8593 8594static int ufs_get_device_desc(struct ufs_hba *hba) 8595{ 8596 struct ufs_dev_info *dev_info = &hba->dev_info; 8597 struct Scsi_Host *shost = hba->host; 8598 int err; 8599 u8 model_index; 8600 u8 *desc_buf; 8601 8602 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8603 if (!desc_buf) { 8604 err = -ENOMEM; 8605 goto out; 8606 } 8607 8608 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf, 8609 QUERY_DESC_MAX_SIZE); 8610 if (err) { 8611 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n", 8612 __func__, err); 8613 goto out; 8614 } 8615 8616 /* 8617 * getting vendor (manufacturerID) and Bank Index in big endian 8618 * format 8619 */ 8620 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 | 8621 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1]; 8622 8623 /* getting Specification Version in big endian format */ 8624 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 | 8625 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; 8626 dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; 8627 8628 /* 8629 * According to the UFS standard, the UFS device queue depth 8630 * (bQueueDepth) must be in the range 1..255 if the shared queueing 8631 * architecture is supported. bQueueDepth is zero if the shared queueing 8632 * architecture is not supported. 8633 */ 8634 if (dev_info->bqueuedepth) 8635 shost->cmd_per_lun = min(hba->nutrs, dev_info->bqueuedepth) - 8636 UFSHCD_NUM_RESERVED; 8637 else 8638 shost->cmd_per_lun = shost->can_queue; 8639 8640 dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP]; 8641 8642 dev_info->hid_sup = get_unaligned_be32(desc_buf + 8643 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP) & 8644 UFS_DEV_HID_SUPPORT; 8645 8646 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; 8647 8648 err = ufshcd_read_string_desc(hba, model_index, 8649 &dev_info->model, SD_ASCII_STD); 8650 if (err < 0) { 8651 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n", 8652 __func__, err); 8653 goto out; 8654 } 8655 8656 /* Generate unique device ID */ 8657 dev_info->device_id = ufshcd_create_device_id(hba, desc_buf); 8658 8659 hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] + 8660 desc_buf[DEVICE_DESC_PARAM_NUM_WLU]; 8661 8662 ufs_fixup_device_setup(hba); 8663 8664 ufshcd_wb_probe(hba, desc_buf); 8665 8666 ufshcd_temp_notif_probe(hba, desc_buf); 8667 8668 if (dev_info->wspecversion >= 0x410) { 8669 hba->critical_health_count = 0; 8670 ufshcd_enable_ee(hba, MASK_EE_HEALTH_CRITICAL); 8671 } 8672 8673 ufs_init_rtc(hba, desc_buf); 8674 8675 ufshcd_device_lvl_exception_probe(hba, desc_buf); 8676 8677 /* 8678 * ufshcd_read_string_desc returns size of the string 8679 * reset the error value 8680 */ 8681 err = 0; 8682 8683out: 8684 kfree(desc_buf); 8685 return err; 8686} 8687 8688static void ufs_put_device_desc(struct ufs_hba *hba) 8689{ 8690 struct ufs_dev_info *dev_info = &hba->dev_info; 8691 8692 kfree(dev_info->model); 8693 dev_info->model = NULL; 8694 kfree(dev_info->device_id); 8695 dev_info->device_id = NULL; 8696} 8697 8698/** 8699 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is 8700 * less than device PA_TACTIVATE time. 8701 * @hba: per-adapter instance 8702 * 8703 * Some UFS devices require host PA_TACTIVATE to be lower than device 8704 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk 8705 * for such devices. 8706 * 8707 * Return: zero on success, non-zero error value on failure. 8708 */ 8709static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba) 8710{ 8711 int ret = 0; 8712 u32 granularity, peer_granularity; 8713 u32 pa_tactivate, peer_pa_tactivate; 8714 u32 pa_tactivate_us, peer_pa_tactivate_us; 8715 static const u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100}; 8716 8717 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY), 8718 &granularity); 8719 if (ret) 8720 goto out; 8721 8722 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY), 8723 &peer_granularity); 8724 if (ret) 8725 goto out; 8726 8727 if ((granularity < PA_GRANULARITY_MIN_VAL) || 8728 (granularity > PA_GRANULARITY_MAX_VAL)) { 8729 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d", 8730 __func__, granularity); 8731 return -EINVAL; 8732 } 8733 8734 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) || 8735 (peer_granularity > PA_GRANULARITY_MAX_VAL)) { 8736 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d", 8737 __func__, peer_granularity); 8738 return -EINVAL; 8739 } 8740 8741 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate); 8742 if (ret) 8743 goto out; 8744 8745 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE), 8746 &peer_pa_tactivate); 8747 if (ret) 8748 goto out; 8749 8750 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1]; 8751 peer_pa_tactivate_us = peer_pa_tactivate * 8752 gran_to_us_table[peer_granularity - 1]; 8753 8754 if (pa_tactivate_us >= peer_pa_tactivate_us) { 8755 u32 new_peer_pa_tactivate; 8756 8757 new_peer_pa_tactivate = pa_tactivate_us / 8758 gran_to_us_table[peer_granularity - 1]; 8759 new_peer_pa_tactivate++; 8760 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 8761 new_peer_pa_tactivate); 8762 } 8763 8764out: 8765 return ret; 8766} 8767 8768/** 8769 * ufshcd_quirk_override_pa_h8time - Ensures proper adjustment of PA_HIBERN8TIME. 8770 * @hba: per-adapter instance 8771 * 8772 * Some UFS devices require specific adjustments to the PA_HIBERN8TIME parameter 8773 * to ensure proper hibernation timing. This function retrieves the current 8774 * PA_HIBERN8TIME value and increments it by 100us. 8775 */ 8776static void ufshcd_quirk_override_pa_h8time(struct ufs_hba *hba) 8777{ 8778 u32 pa_h8time; 8779 int ret; 8780 8781 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), &pa_h8time); 8782 if (ret) { 8783 dev_err(hba->dev, "Failed to get PA_HIBERN8TIME: %d\n", ret); 8784 return; 8785 } 8786 8787 /* Increment by 1 to increase hibernation time by 100 µs */ 8788 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), pa_h8time + 1); 8789 if (ret) 8790 dev_err(hba->dev, "Failed updating PA_HIBERN8TIME: %d\n", ret); 8791} 8792 8793static void ufshcd_tune_unipro_params(struct ufs_hba *hba) 8794{ 8795 ufshcd_vops_apply_dev_quirks(hba); 8796 8797 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE) 8798 /* set 1ms timeout for PA_TACTIVATE */ 8799 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10); 8800 8801 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE) 8802 ufshcd_quirk_tune_host_pa_tactivate(hba); 8803 8804 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_HIBER8TIME) 8805 ufshcd_quirk_override_pa_h8time(hba); 8806} 8807 8808static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) 8809{ 8810 hba->ufs_stats.hibern8_exit_cnt = 0; 8811 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 8812 hba->req_abort_count = 0; 8813} 8814 8815static int ufshcd_device_geo_params_init(struct ufs_hba *hba) 8816{ 8817 int err; 8818 u8 *desc_buf; 8819 8820 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8821 if (!desc_buf) { 8822 err = -ENOMEM; 8823 goto out; 8824 } 8825 8826 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0, 8827 desc_buf, QUERY_DESC_MAX_SIZE); 8828 if (err) { 8829 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n", 8830 __func__, err); 8831 goto out; 8832 } 8833 8834 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1) 8835 hba->dev_info.max_lu_supported = 32; 8836 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0) 8837 hba->dev_info.max_lu_supported = 8; 8838 8839 hba->dev_info.rpmb_io_size = desc_buf[GEOMETRY_DESC_PARAM_RPMB_RW_SIZE]; 8840 8841out: 8842 kfree(desc_buf); 8843 return err; 8844} 8845 8846struct ufs_ref_clk { 8847 unsigned long freq_hz; 8848 enum ufs_ref_clk_freq val; 8849}; 8850 8851static const struct ufs_ref_clk ufs_ref_clk_freqs[] = { 8852 {19200000, REF_CLK_FREQ_19_2_MHZ}, 8853 {26000000, REF_CLK_FREQ_26_MHZ}, 8854 {38400000, REF_CLK_FREQ_38_4_MHZ}, 8855 {52000000, REF_CLK_FREQ_52_MHZ}, 8856 {0, REF_CLK_FREQ_INVAL}, 8857}; 8858 8859static enum ufs_ref_clk_freq 8860ufs_get_bref_clk_from_hz(unsigned long freq) 8861{ 8862 int i; 8863 8864 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++) 8865 if (ufs_ref_clk_freqs[i].freq_hz == freq) 8866 return ufs_ref_clk_freqs[i].val; 8867 8868 return REF_CLK_FREQ_INVAL; 8869} 8870 8871void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk) 8872{ 8873 unsigned long freq; 8874 8875 freq = clk_get_rate(refclk); 8876 8877 hba->dev_ref_clk_freq = 8878 ufs_get_bref_clk_from_hz(freq); 8879 8880 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL) 8881 dev_err(hba->dev, 8882 "invalid ref_clk setting = %ld\n", freq); 8883} 8884 8885static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba) 8886{ 8887 int err; 8888 u32 ref_clk; 8889 u32 freq = hba->dev_ref_clk_freq; 8890 8891 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 8892 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk); 8893 8894 if (err) { 8895 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n", 8896 err); 8897 goto out; 8898 } 8899 8900 if (ref_clk == freq) 8901 goto out; /* nothing to update */ 8902 8903 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8904 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq); 8905 8906 if (err) { 8907 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n", 8908 ufs_ref_clk_freqs[freq].freq_hz); 8909 goto out; 8910 } 8911 8912 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n", 8913 ufs_ref_clk_freqs[freq].freq_hz); 8914 8915out: 8916 return err; 8917} 8918 8919static int ufshcd_device_params_init(struct ufs_hba *hba) 8920{ 8921 bool flag; 8922 int ret; 8923 8924 /* Init UFS geometry descriptor related parameters */ 8925 ret = ufshcd_device_geo_params_init(hba); 8926 if (ret) 8927 goto out; 8928 8929 /* Check and apply UFS device quirks */ 8930 ret = ufs_get_device_desc(hba); 8931 if (ret) { 8932 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n", 8933 __func__, ret); 8934 goto out; 8935 } 8936 8937 ufshcd_set_rtt(hba); 8938 8939 ufshcd_get_ref_clk_gating_wait(hba); 8940 8941 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG, 8942 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag)) 8943 hba->dev_info.f_power_on_wp_en = flag; 8944 8945 /* Probe maximum power mode co-supported by both UFS host and device */ 8946 if (ufshcd_get_max_pwr_mode(hba)) 8947 dev_err(hba->dev, 8948 "%s: Failed getting max supported power mode\n", 8949 __func__); 8950out: 8951 return ret; 8952} 8953 8954static void ufshcd_set_timestamp_attr(struct ufs_hba *hba) 8955{ 8956 int err; 8957 struct ufs_query_req *request = NULL; 8958 struct ufs_query_res *response = NULL; 8959 struct ufs_dev_info *dev_info = &hba->dev_info; 8960 struct utp_upiu_query_v4_0 *upiu_data; 8961 8962 if (dev_info->wspecversion < 0x400 || 8963 hba->dev_quirks & UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT) 8964 return; 8965 8966 ufshcd_dev_man_lock(hba); 8967 8968 ufshcd_init_query(hba, &request, &response, 8969 UPIU_QUERY_OPCODE_WRITE_ATTR, 8970 QUERY_ATTR_IDN_TIMESTAMP, 0, 0); 8971 8972 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 8973 8974 upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req; 8975 8976 put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3); 8977 8978 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); 8979 8980 if (err) 8981 dev_err(hba->dev, "%s: failed to set timestamp %d\n", 8982 __func__, err); 8983 8984 ufshcd_dev_man_unlock(hba); 8985} 8986 8987/** 8988 * ufshcd_add_lus - probe and add UFS logical units 8989 * @hba: per-adapter instance 8990 * 8991 * Return: 0 upon success; < 0 upon failure. 8992 */ 8993static int ufshcd_add_lus(struct ufs_hba *hba) 8994{ 8995 int ret; 8996 8997 /* Add required well known logical units to scsi mid layer */ 8998 ret = ufshcd_scsi_add_wlus(hba); 8999 if (ret) 9000 goto out; 9001 9002 /* Initialize devfreq after UFS device is detected */ 9003 if (ufshcd_is_clkscaling_supported(hba)) { 9004 memcpy(&hba->clk_scaling.saved_pwr_info, 9005 &hba->pwr_info, 9006 sizeof(struct ufs_pa_layer_attr)); 9007 hba->clk_scaling.is_allowed = true; 9008 9009 ret = ufshcd_devfreq_init(hba); 9010 if (ret) 9011 goto out; 9012 9013 hba->clk_scaling.is_enabled = true; 9014 ufshcd_init_clk_scaling_sysfs(hba); 9015 } 9016 9017 /* 9018 * The RTC update code accesses the hba->ufs_device_wlun->sdev_gendev 9019 * pointer and hence must only be started after the WLUN pointer has 9020 * been initialized by ufshcd_scsi_add_wlus(). 9021 */ 9022 schedule_delayed_work(&hba->ufs_rtc_update_work, 9023 msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); 9024 9025 ufs_bsg_probe(hba); 9026 scsi_scan_host(hba->host); 9027 ufs_rpmb_probe(hba); 9028 9029out: 9030 return ret; 9031} 9032 9033/* SDB - Single Doorbell */ 9034static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs) 9035{ 9036 size_t ucdl_size, utrdl_size; 9037 9038 ucdl_size = ufshcd_get_ucd_size(hba) * nutrs; 9039 dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr, 9040 hba->ucdl_dma_addr); 9041 9042 utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; 9043 dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, 9044 hba->utrdl_dma_addr); 9045} 9046 9047static int ufshcd_alloc_mcq(struct ufs_hba *hba) 9048{ 9049 int ret; 9050 int old_nutrs = hba->nutrs; 9051 9052 ret = ufshcd_get_hba_mac(hba); 9053 if (ret < 0) 9054 return ret; 9055 9056 hba->nutrs = ret; 9057 ret = ufshcd_mcq_init(hba); 9058 if (ret) 9059 goto err; 9060 9061 /* 9062 * Previously allocated memory for nutrs may not be enough in MCQ mode. 9063 * Number of supported tags in MCQ mode may be larger than SDB mode. 9064 */ 9065 if (hba->nutrs != old_nutrs) { 9066 ufshcd_release_sdb_queue(hba, old_nutrs); 9067 ret = ufshcd_memory_alloc(hba); 9068 if (ret) 9069 goto err; 9070 ufshcd_host_memory_configure(hba); 9071 } 9072 9073 ret = ufshcd_mcq_memory_alloc(hba); 9074 if (ret) 9075 goto err; 9076 9077 hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 9078 9079 return 0; 9080err: 9081 hba->nutrs = old_nutrs; 9082 return ret; 9083} 9084 9085static void ufshcd_config_mcq(struct ufs_hba *hba) 9086{ 9087 int ret; 9088 9089 ret = ufshcd_mcq_vops_config_esi(hba); 9090 hba->mcq_esi_enabled = !ret; 9091 dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : ""); 9092 9093 ufshcd_mcq_make_queues_operational(hba); 9094 ufshcd_mcq_config_mac(hba, hba->nutrs); 9095 9096 dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n", 9097 hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT], 9098 hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL], 9099 hba->nutrs); 9100} 9101 9102static int ufshcd_post_device_init(struct ufs_hba *hba) 9103{ 9104 int ret; 9105 9106 ufshcd_tune_unipro_params(hba); 9107 9108 /* UFS device is also active now */ 9109 ufshcd_set_ufs_dev_active(hba); 9110 ufshcd_force_reset_auto_bkops(hba); 9111 9112 ufshcd_set_timestamp_attr(hba); 9113 9114 if (!hba->max_pwr_info.is_valid) 9115 return 0; 9116 9117 /* 9118 * Set the right value to bRefClkFreq before attempting to 9119 * switch to HS gears. 9120 */ 9121 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL) 9122 ufshcd_set_dev_ref_clk(hba); 9123 /* Gear up to HS gear. */ 9124 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); 9125 if (ret) { 9126 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", 9127 __func__, ret); 9128 return ret; 9129 } 9130 9131 return 0; 9132} 9133 9134static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) 9135{ 9136 int ret; 9137 9138 WARN_ON_ONCE(!hba->scsi_host_added); 9139 9140 hba->ufshcd_state = UFSHCD_STATE_RESET; 9141 9142 ret = ufshcd_link_startup(hba); 9143 if (ret) 9144 return ret; 9145 9146 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) 9147 return ret; 9148 9149 /* Debug counters initialization */ 9150 ufshcd_clear_dbg_ufs_stats(hba); 9151 9152 /* UniPro link is active now */ 9153 ufshcd_set_link_active(hba); 9154 9155 /* Reconfigure MCQ upon reset */ 9156 if (hba->mcq_enabled && !init_dev_params) { 9157 ufshcd_config_mcq(hba); 9158 ufshcd_mcq_enable(hba); 9159 } 9160 9161 /* Verify device initialization by sending NOP OUT UPIU */ 9162 ret = ufshcd_verify_dev_init(hba); 9163 if (ret) 9164 return ret; 9165 9166 /* Initiate UFS initialization, and waiting until completion */ 9167 ret = ufshcd_complete_dev_init(hba); 9168 if (ret) 9169 return ret; 9170 9171 /* 9172 * Initialize UFS device parameters used by driver, these 9173 * parameters are associated with UFS descriptors. 9174 */ 9175 if (init_dev_params) { 9176 ret = ufshcd_device_params_init(hba); 9177 if (ret) 9178 return ret; 9179 if (is_mcq_supported(hba) && 9180 hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) { 9181 ufshcd_config_mcq(hba); 9182 ufshcd_mcq_enable(hba); 9183 } 9184 } 9185 9186 return ufshcd_post_device_init(hba); 9187} 9188 9189/** 9190 * ufshcd_probe_hba - probe hba to detect device and initialize it 9191 * @hba: per-adapter instance 9192 * @init_dev_params: whether or not to call ufshcd_device_params_init(). 9193 * 9194 * Execute link-startup and verify device initialization 9195 * 9196 * Return: 0 upon success; < 0 upon failure. 9197 */ 9198static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) 9199{ 9200 int ret; 9201 9202 if (!hba->pm_op_in_progress && 9203 (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) { 9204 /* Reset the device and controller before doing reinit */ 9205 ufshcd_device_reset(hba); 9206 ufs_put_device_desc(hba); 9207 ufshcd_hba_stop(hba); 9208 ret = ufshcd_hba_enable(hba); 9209 if (ret) { 9210 dev_err(hba->dev, "Host controller enable failed\n"); 9211 ufshcd_print_evt_hist(hba); 9212 ufshcd_print_host_state(hba); 9213 return ret; 9214 } 9215 9216 /* Reinit the device */ 9217 ret = ufshcd_device_init(hba, init_dev_params); 9218 if (ret) 9219 return ret; 9220 } 9221 9222 ufshcd_print_pwr_info(hba); 9223 9224 /* 9225 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec) 9226 * and for removable UFS card as well, hence always set the parameter. 9227 * Note: Error handler may issue the device reset hence resetting 9228 * bActiveICCLevel as well so it is always safe to set this here. 9229 */ 9230 ufshcd_set_active_icc_lvl(hba); 9231 9232 /* Enable UFS Write Booster if supported */ 9233 ufshcd_configure_wb(hba); 9234 9235 if (hba->ee_usr_mask) 9236 ufshcd_write_ee_control(hba); 9237 ufshcd_configure_auto_hibern8(hba); 9238 9239 return 0; 9240} 9241 9242/** 9243 * ufshcd_async_scan - asynchronous execution for probing hba 9244 * @data: data pointer to pass to this function 9245 * @cookie: cookie data 9246 */ 9247static void ufshcd_async_scan(void *data, async_cookie_t cookie) 9248{ 9249 struct ufs_hba *hba = (struct ufs_hba *)data; 9250 ktime_t probe_start; 9251 int ret; 9252 9253 down(&hba->host_sem); 9254 /* Initialize hba, detect and initialize UFS device */ 9255 probe_start = ktime_get(); 9256 ret = ufshcd_probe_hba(hba, true); 9257 ufshcd_process_probe_result(hba, probe_start, ret); 9258 up(&hba->host_sem); 9259 if (ret) 9260 goto out; 9261 9262 /* Probe and add UFS logical units */ 9263 ret = ufshcd_add_lus(hba); 9264 9265out: 9266 pm_runtime_put_sync(hba->dev); 9267 9268 if (ret) 9269 dev_err(hba->dev, "%s failed: %d\n", __func__, ret); 9270} 9271 9272static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) 9273{ 9274 struct ufs_hba *hba = shost_priv(scmd->device->host); 9275 9276 if (!hba->system_suspending) { 9277 /* Activate the error handler in the SCSI core. */ 9278 return SCSI_EH_NOT_HANDLED; 9279 } 9280 9281 /* 9282 * If we get here we know that no TMFs are outstanding and also that 9283 * the only pending command is a START STOP UNIT command. Handle the 9284 * timeout of that command directly to prevent a deadlock between 9285 * ufshcd_set_dev_pwr_mode() and ufshcd_err_handler(). 9286 */ 9287 ufshcd_link_recovery(hba); 9288 dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n", 9289 __func__, hba->outstanding_tasks); 9290 9291 return scsi_host_busy(hba->host) ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; 9292} 9293 9294static const struct attribute_group *ufshcd_driver_groups[] = { 9295 &ufs_sysfs_unit_descriptor_group, 9296 &ufs_sysfs_lun_attributes_group, 9297 NULL, 9298}; 9299 9300static struct ufs_hba_variant_params ufs_hba_vps = { 9301 .hba_enable_delay_us = 1000, 9302 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40), 9303 .devfreq_profile.polling_ms = 100, 9304 .devfreq_profile.target = ufshcd_devfreq_target, 9305 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status, 9306 .ondemand_data.upthreshold = 70, 9307 .ondemand_data.downdifferential = 5, 9308}; 9309 9310static const struct scsi_host_template ufshcd_driver_template = { 9311 .module = THIS_MODULE, 9312 .name = UFSHCD, 9313 .proc_name = UFSHCD, 9314 .map_queues = ufshcd_map_queues, 9315 .cmd_size = sizeof(struct ufshcd_lrb), 9316 .init_cmd_priv = ufshcd_init_cmd_priv, 9317 .queuecommand = ufshcd_queuecommand, 9318 .queue_reserved_command = ufshcd_queue_reserved_command, 9319 .nr_reserved_cmds = UFSHCD_NUM_RESERVED, 9320 .mq_poll = ufshcd_poll, 9321 .sdev_init = ufshcd_sdev_init, 9322 .sdev_configure = ufshcd_sdev_configure, 9323 .sdev_destroy = ufshcd_sdev_destroy, 9324 .change_queue_depth = ufshcd_change_queue_depth, 9325 .eh_abort_handler = ufshcd_abort, 9326 .eh_device_reset_handler = ufshcd_eh_device_reset_handler, 9327 .eh_host_reset_handler = ufshcd_eh_host_reset_handler, 9328 .eh_timed_out = ufshcd_eh_timed_out, 9329 .this_id = -1, 9330 .sg_tablesize = SG_ALL, 9331 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX, 9332 .max_sectors = SZ_1M / SECTOR_SIZE, 9333 .max_host_blocked = 1, 9334 .track_queue_depth = 1, 9335 .skip_settle_delay = 1, 9336 .sdev_groups = ufshcd_driver_groups, 9337}; 9338 9339static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, 9340 int ua) 9341{ 9342 int ret; 9343 9344 if (!vreg) 9345 return 0; 9346 9347 /* 9348 * "set_load" operation shall be required on those regulators 9349 * which specifically configured current limitation. Otherwise 9350 * zero max_uA may cause unexpected behavior when regulator is 9351 * enabled or set as high power mode. 9352 */ 9353 if (!vreg->max_uA) 9354 return 0; 9355 9356 ret = regulator_set_load(vreg->reg, ua); 9357 if (ret < 0) { 9358 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n", 9359 __func__, vreg->name, ua, ret); 9360 } 9361 9362 return ret; 9363} 9364 9365static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba, 9366 struct ufs_vreg *vreg) 9367{ 9368 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA); 9369} 9370 9371static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 9372 struct ufs_vreg *vreg) 9373{ 9374 if (!vreg) 9375 return 0; 9376 9377 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA); 9378} 9379 9380static int ufshcd_config_vreg(struct device *dev, 9381 struct ufs_vreg *vreg, bool on) 9382{ 9383 if (regulator_count_voltages(vreg->reg) <= 0) 9384 return 0; 9385 9386 return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0); 9387} 9388 9389static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg) 9390{ 9391 int ret = 0; 9392 9393 if (!vreg || vreg->enabled) 9394 goto out; 9395 9396 ret = ufshcd_config_vreg(dev, vreg, true); 9397 if (!ret) 9398 ret = regulator_enable(vreg->reg); 9399 9400 if (!ret) 9401 vreg->enabled = true; 9402 else 9403 dev_err(dev, "%s: %s enable failed, err=%d\n", 9404 __func__, vreg->name, ret); 9405out: 9406 return ret; 9407} 9408 9409static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg) 9410{ 9411 int ret = 0; 9412 9413 if (!vreg || !vreg->enabled || vreg->always_on) 9414 goto out; 9415 9416 ret = regulator_disable(vreg->reg); 9417 9418 if (!ret) { 9419 /* ignore errors on applying disable config */ 9420 ufshcd_config_vreg(dev, vreg, false); 9421 vreg->enabled = false; 9422 } else { 9423 dev_err(dev, "%s: %s disable failed, err=%d\n", 9424 __func__, vreg->name, ret); 9425 } 9426out: 9427 return ret; 9428} 9429 9430static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on) 9431{ 9432 int ret = 0; 9433 struct device *dev = hba->dev; 9434 struct ufs_vreg_info *info = &hba->vreg_info; 9435 9436 ret = ufshcd_toggle_vreg(dev, info->vcc, on); 9437 if (ret) 9438 goto out; 9439 9440 ret = ufshcd_toggle_vreg(dev, info->vccq, on); 9441 if (ret) 9442 goto out; 9443 9444 ret = ufshcd_toggle_vreg(dev, info->vccq2, on); 9445 9446out: 9447 if (ret) { 9448 ufshcd_toggle_vreg(dev, info->vccq2, false); 9449 ufshcd_toggle_vreg(dev, info->vccq, false); 9450 ufshcd_toggle_vreg(dev, info->vcc, false); 9451 } 9452 return ret; 9453} 9454 9455static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on) 9456{ 9457 struct ufs_vreg_info *info = &hba->vreg_info; 9458 9459 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on); 9460} 9461 9462int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg) 9463{ 9464 int ret = 0; 9465 9466 if (!vreg) 9467 goto out; 9468 9469 vreg->reg = devm_regulator_get(dev, vreg->name); 9470 if (IS_ERR(vreg->reg)) { 9471 ret = PTR_ERR(vreg->reg); 9472 dev_err(dev, "%s: %s get failed, err=%d\n", 9473 __func__, vreg->name, ret); 9474 } 9475out: 9476 return ret; 9477} 9478EXPORT_SYMBOL_GPL(ufshcd_get_vreg); 9479 9480static int ufshcd_init_vreg(struct ufs_hba *hba) 9481{ 9482 int ret = 0; 9483 struct device *dev = hba->dev; 9484 struct ufs_vreg_info *info = &hba->vreg_info; 9485 9486 ret = ufshcd_get_vreg(dev, info->vcc); 9487 if (ret) 9488 goto out; 9489 9490 ret = ufshcd_get_vreg(dev, info->vccq); 9491 if (!ret) 9492 ret = ufshcd_get_vreg(dev, info->vccq2); 9493out: 9494 return ret; 9495} 9496 9497static int ufshcd_init_hba_vreg(struct ufs_hba *hba) 9498{ 9499 struct ufs_vreg_info *info = &hba->vreg_info; 9500 9501 return ufshcd_get_vreg(hba->dev, info->vdd_hba); 9502} 9503 9504static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) 9505{ 9506 int ret = 0; 9507 struct ufs_clk_info *clki; 9508 struct list_head *head = &hba->clk_list_head; 9509 ktime_t start = ktime_get(); 9510 bool clk_state_changed = false; 9511 9512 if (list_empty(head)) 9513 goto out; 9514 9515 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE); 9516 if (ret) 9517 return ret; 9518 9519 list_for_each_entry(clki, head, list) { 9520 if (!IS_ERR_OR_NULL(clki->clk)) { 9521 /* 9522 * Don't disable clocks which are needed 9523 * to keep the link active. 9524 */ 9525 if (ufshcd_is_link_active(hba) && 9526 clki->keep_link_active) 9527 continue; 9528 9529 clk_state_changed = on ^ clki->enabled; 9530 if (on && !clki->enabled) { 9531 ret = clk_prepare_enable(clki->clk); 9532 if (ret) { 9533 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n", 9534 __func__, clki->name, ret); 9535 goto out; 9536 } 9537 } else if (!on && clki->enabled) { 9538 clk_disable_unprepare(clki->clk); 9539 } 9540 clki->enabled = on; 9541 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__, 9542 clki->name, on ? "en" : "dis"); 9543 } 9544 } 9545 9546 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE); 9547 if (ret) 9548 return ret; 9549 9550 if (!ufshcd_is_clkscaling_supported(hba)) 9551 ufshcd_pm_qos_update(hba, on); 9552out: 9553 if (ret) { 9554 list_for_each_entry(clki, head, list) { 9555 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) 9556 clk_disable_unprepare(clki->clk); 9557 } 9558 } else if (!ret && on && hba->clk_gating.is_initialized) { 9559 scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) 9560 hba->clk_gating.state = CLKS_ON; 9561 trace_ufshcd_clk_gating(hba, 9562 hba->clk_gating.state); 9563 } 9564 9565 if (clk_state_changed) 9566 trace_ufshcd_profile_clk_gating(hba, 9567 (on ? "on" : "off"), 9568 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 9569 return ret; 9570} 9571 9572static enum ufs_ref_clk_freq ufshcd_parse_ref_clk_property(struct ufs_hba *hba) 9573{ 9574 u32 freq; 9575 int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq); 9576 9577 if (ret) { 9578 dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret); 9579 return REF_CLK_FREQ_INVAL; 9580 } 9581 9582 return ufs_get_bref_clk_from_hz(freq); 9583} 9584 9585static int ufshcd_init_clocks(struct ufs_hba *hba) 9586{ 9587 int ret = 0; 9588 struct ufs_clk_info *clki; 9589 struct device *dev = hba->dev; 9590 struct list_head *head = &hba->clk_list_head; 9591 9592 if (list_empty(head)) 9593 goto out; 9594 9595 list_for_each_entry(clki, head, list) { 9596 if (!clki->name) 9597 continue; 9598 9599 clki->clk = devm_clk_get(dev, clki->name); 9600 if (IS_ERR(clki->clk)) { 9601 ret = PTR_ERR(clki->clk); 9602 dev_err(dev, "%s: %s clk get failed, %d\n", 9603 __func__, clki->name, ret); 9604 goto out; 9605 } 9606 9607 /* 9608 * Parse device ref clk freq as per device tree "ref_clk". 9609 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL 9610 * in ufshcd_alloc_host(). 9611 */ 9612 if (!strcmp(clki->name, "ref_clk")) 9613 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk); 9614 9615 if (clki->max_freq) { 9616 ret = clk_set_rate(clki->clk, clki->max_freq); 9617 if (ret) { 9618 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 9619 __func__, clki->name, 9620 clki->max_freq, ret); 9621 goto out; 9622 } 9623 clki->curr_freq = clki->max_freq; 9624 } 9625 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__, 9626 clki->name, clk_get_rate(clki->clk)); 9627 } 9628 9629 /* Set Max. frequency for all clocks */ 9630 if (hba->use_pm_opp) { 9631 ret = ufshcd_opp_set_rate(hba, ULONG_MAX); 9632 if (ret) { 9633 dev_err(hba->dev, "%s: failed to set OPP: %d", __func__, 9634 ret); 9635 goto out; 9636 } 9637 } 9638 9639out: 9640 return ret; 9641} 9642 9643static int ufshcd_variant_hba_init(struct ufs_hba *hba) 9644{ 9645 int err = 0; 9646 9647 if (!hba->vops) 9648 goto out; 9649 9650 err = ufshcd_vops_init(hba); 9651 if (err) 9652 dev_err_probe(hba->dev, err, 9653 "%s: variant %s init failed with err %d\n", 9654 __func__, ufshcd_get_var_name(hba), err); 9655out: 9656 return err; 9657} 9658 9659static void ufshcd_variant_hba_exit(struct ufs_hba *hba) 9660{ 9661 if (!hba->vops) 9662 return; 9663 9664 ufshcd_vops_exit(hba); 9665} 9666 9667static int ufshcd_hba_init(struct ufs_hba *hba) 9668{ 9669 int err; 9670 9671 /* 9672 * Handle host controller power separately from the UFS device power 9673 * rails as it will help controlling the UFS host controller power 9674 * collapse easily which is different than UFS device power collapse. 9675 * Also, enable the host controller power before we go ahead with rest 9676 * of the initialization here. 9677 */ 9678 err = ufshcd_init_hba_vreg(hba); 9679 if (err) 9680 goto out; 9681 9682 err = ufshcd_setup_hba_vreg(hba, true); 9683 if (err) 9684 goto out; 9685 9686 err = ufshcd_init_clocks(hba); 9687 if (err) 9688 goto out_disable_hba_vreg; 9689 9690 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL) 9691 hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba); 9692 9693 err = ufshcd_setup_clocks(hba, true); 9694 if (err) 9695 goto out_disable_hba_vreg; 9696 9697 err = ufshcd_init_vreg(hba); 9698 if (err) 9699 goto out_disable_clks; 9700 9701 err = ufshcd_setup_vreg(hba, true); 9702 if (err) 9703 goto out_disable_clks; 9704 9705 err = ufshcd_variant_hba_init(hba); 9706 if (err) 9707 goto out_disable_vreg; 9708 9709 ufs_debugfs_hba_init(hba); 9710 ufs_fault_inject_hba_init(hba); 9711 9712 hba->is_powered = true; 9713 goto out; 9714 9715out_disable_vreg: 9716 ufshcd_setup_vreg(hba, false); 9717out_disable_clks: 9718 ufshcd_setup_clocks(hba, false); 9719out_disable_hba_vreg: 9720 ufshcd_setup_hba_vreg(hba, false); 9721out: 9722 return err; 9723} 9724 9725static void ufshcd_hba_exit(struct ufs_hba *hba) 9726{ 9727 if (hba->is_powered) { 9728 ufshcd_pm_qos_exit(hba); 9729 ufshcd_exit_clk_scaling(hba); 9730 ufshcd_exit_clk_gating(hba); 9731 if (hba->eh_wq) 9732 destroy_workqueue(hba->eh_wq); 9733 ufs_debugfs_hba_exit(hba); 9734 ufshcd_variant_hba_exit(hba); 9735 ufshcd_setup_vreg(hba, false); 9736 ufshcd_setup_clocks(hba, false); 9737 ufshcd_setup_hba_vreg(hba, false); 9738 hba->is_powered = false; 9739 ufs_put_device_desc(hba); 9740 } 9741} 9742 9743static int ufshcd_execute_start_stop(struct scsi_device *sdev, 9744 enum ufs_dev_pwr_mode pwr_mode, 9745 struct scsi_sense_hdr *sshdr) 9746{ 9747 const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 }; 9748 struct scsi_failure failure_defs[] = { 9749 { 9750 .allowed = 2, 9751 .result = SCMD_FAILURE_RESULT_ANY, 9752 }, 9753 }; 9754 struct scsi_failures failures = { 9755 .failure_definitions = failure_defs, 9756 }; 9757 const struct scsi_exec_args args = { 9758 .failures = &failures, 9759 .sshdr = sshdr, 9760 .req_flags = BLK_MQ_REQ_PM, 9761 .scmd_flags = SCMD_FAIL_IF_RECOVERING, 9762 }; 9763 9764 return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL, 9765 /*bufflen=*/0, /*timeout=*/10 * HZ, /*retries=*/0, 9766 &args); 9767} 9768 9769/** 9770 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device 9771 * power mode 9772 * @hba: per adapter instance 9773 * @pwr_mode: device power mode to set 9774 * 9775 * Return: 0 if requested power mode is set successfully; 9776 * < 0 if failed to set the requested power mode. 9777 */ 9778static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, 9779 enum ufs_dev_pwr_mode pwr_mode) 9780{ 9781 struct scsi_sense_hdr sshdr; 9782 struct scsi_device *sdp; 9783 unsigned long flags; 9784 int ret; 9785 9786 spin_lock_irqsave(hba->host->host_lock, flags); 9787 sdp = hba->ufs_device_wlun; 9788 if (sdp && scsi_device_online(sdp)) 9789 ret = scsi_device_get(sdp); 9790 else 9791 ret = -ENODEV; 9792 spin_unlock_irqrestore(hba->host->host_lock, flags); 9793 9794 if (ret) 9795 return ret; 9796 9797 /* 9798 * If scsi commands fail, the scsi mid-layer schedules scsi error- 9799 * handling, which would wait for host to be resumed. Since we know 9800 * we are functional while we are here, skip host resume in error 9801 * handling context. 9802 */ 9803 hba->host->eh_noresume = 1; 9804 9805 /* 9806 * Current function would be generally called from the power management 9807 * callbacks hence set the RQF_PM flag so that it doesn't resume the 9808 * already suspended childs. 9809 */ 9810 ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); 9811 if (ret) { 9812 sdev_printk(KERN_WARNING, sdp, 9813 "START_STOP failed for power mode: %d, result %x\n", 9814 pwr_mode, ret); 9815 if (ret > 0) { 9816 if (scsi_sense_valid(&sshdr)) 9817 scsi_print_sense_hdr(sdp, NULL, &sshdr); 9818 ret = -EIO; 9819 } 9820 } else { 9821 hba->curr_dev_pwr_mode = pwr_mode; 9822 } 9823 9824 scsi_device_put(sdp); 9825 hba->host->eh_noresume = 0; 9826 return ret; 9827} 9828 9829static int ufshcd_link_state_transition(struct ufs_hba *hba, 9830 enum uic_link_state req_link_state, 9831 bool check_for_bkops) 9832{ 9833 int ret = 0; 9834 9835 if (req_link_state == hba->uic_link_state) 9836 return 0; 9837 9838 if (req_link_state == UIC_LINK_HIBERN8_STATE) { 9839 ret = ufshcd_uic_hibern8_enter(hba); 9840 if (!ret) { 9841 ufshcd_set_link_hibern8(hba); 9842 } else { 9843 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 9844 __func__, ret); 9845 goto out; 9846 } 9847 } 9848 /* 9849 * If autobkops is enabled, link can't be turned off because 9850 * turning off the link would also turn off the device, except in the 9851 * case of DeepSleep where the device is expected to remain powered. 9852 */ 9853 else if ((req_link_state == UIC_LINK_OFF_STATE) && 9854 (!check_for_bkops || !hba->auto_bkops_enabled)) { 9855 /* 9856 * Let's make sure that link is in low power mode, we are doing 9857 * this currently by putting the link in Hibern8. Otherway to 9858 * put the link in low power mode is to send the DME end point 9859 * to device and then send the DME reset command to local 9860 * unipro. But putting the link in hibern8 is much faster. 9861 * 9862 * Note also that putting the link in Hibern8 is a requirement 9863 * for entering DeepSleep. 9864 */ 9865 ret = ufshcd_uic_hibern8_enter(hba); 9866 if (ret) { 9867 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 9868 __func__, ret); 9869 goto out; 9870 } 9871 /* 9872 * Change controller state to "reset state" which 9873 * should also put the link in off/reset state 9874 */ 9875 ufshcd_hba_stop(hba); 9876 /* 9877 * TODO: Check if we need any delay to make sure that 9878 * controller is reset 9879 */ 9880 ufshcd_set_link_off(hba); 9881 } 9882 9883out: 9884 return ret; 9885} 9886 9887static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) 9888{ 9889 bool vcc_off = false; 9890 9891 /* 9892 * It seems some UFS devices may keep drawing more than sleep current 9893 * (atleast for 500us) from UFS rails (especially from VCCQ rail). 9894 * To avoid this situation, add 2ms delay before putting these UFS 9895 * rails in LPM mode. 9896 */ 9897 if (!ufshcd_is_link_active(hba) && 9898 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM) 9899 usleep_range(2000, 2100); 9900 9901 /* 9902 * If UFS device is either in UFS_Sleep turn off VCC rail to save some 9903 * power. 9904 * 9905 * If UFS device and link is in OFF state, all power supplies (VCC, 9906 * VCCQ, VCCQ2) can be turned off if power on write protect is not 9907 * required. If UFS link is inactive (Hibern8 or OFF state) and device 9908 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode. 9909 * 9910 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway 9911 * in low power state which would save some power. 9912 * 9913 * If Write Booster is enabled and the device needs to flush the WB 9914 * buffer OR if bkops status is urgent for WB, keep Vcc on. 9915 */ 9916 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 9917 !hba->dev_info.is_lu_power_on_wp) { 9918 ufshcd_setup_vreg(hba, false); 9919 vcc_off = true; 9920 } else if (!ufshcd_is_ufs_dev_active(hba)) { 9921 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 9922 vcc_off = true; 9923 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) { 9924 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 9925 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2); 9926 } 9927 } 9928 9929 /* 9930 * All UFS devices require delay after VCC power rail is turned-off. 9931 */ 9932 if (vcc_off && hba->vreg_info.vcc && !hba->vreg_info.vcc->always_on) 9933 usleep_range(hba->vcc_off_delay_us, 9934 hba->vcc_off_delay_us + 100); 9935} 9936 9937#ifdef CONFIG_PM 9938static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) 9939{ 9940 int ret = 0; 9941 9942 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 9943 !hba->dev_info.is_lu_power_on_wp) { 9944 ret = ufshcd_setup_vreg(hba, true); 9945 } else if (!ufshcd_is_ufs_dev_active(hba)) { 9946 if (!ufshcd_is_link_active(hba)) { 9947 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 9948 if (ret) 9949 goto vcc_disable; 9950 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 9951 if (ret) 9952 goto vccq_lpm; 9953 } 9954 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true); 9955 } 9956 goto out; 9957 9958vccq_lpm: 9959 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 9960vcc_disable: 9961 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 9962out: 9963 return ret; 9964} 9965#endif /* CONFIG_PM */ 9966 9967static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba) 9968{ 9969 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) 9970 ufshcd_setup_hba_vreg(hba, false); 9971} 9972 9973static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba) 9974{ 9975 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) 9976 ufshcd_setup_hba_vreg(hba, true); 9977} 9978 9979static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) 9980{ 9981 int ret = 0; 9982 bool check_for_bkops; 9983 enum ufs_pm_level pm_lvl; 9984 enum ufs_dev_pwr_mode req_dev_pwr_mode; 9985 enum uic_link_state req_link_state; 9986 9987 hba->pm_op_in_progress = true; 9988 if (pm_op != UFS_SHUTDOWN_PM) { 9989 pm_lvl = pm_op == UFS_RUNTIME_PM ? 9990 hba->rpm_lvl : hba->spm_lvl; 9991 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl); 9992 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl); 9993 } else { 9994 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE; 9995 req_link_state = UIC_LINK_OFF_STATE; 9996 } 9997 9998 /* 9999 * If we can't transition into any of the low power modes 10000 * just gate the clocks. 10001 */ 10002 ufshcd_hold(hba); 10003 hba->clk_gating.is_suspended = true; 10004 10005 if (ufshcd_is_clkscaling_supported(hba)) 10006 ufshcd_clk_scaling_suspend(hba, true); 10007 10008 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && 10009 req_link_state == UIC_LINK_ACTIVE_STATE) { 10010 ufshcd_disable_auto_bkops(hba); 10011 flush_work(&hba->eeh_work); 10012 goto vops_suspend; 10013 } 10014 10015 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) && 10016 (req_link_state == hba->uic_link_state)) 10017 goto enable_scaling; 10018 10019 /* UFS device & link must be active before we enter in this function */ 10020 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) { 10021 /* Wait err handler finish or trigger err recovery */ 10022 if (!ufshcd_eh_in_progress(hba)) 10023 ufshcd_force_error_recovery(hba); 10024 ret = -EBUSY; 10025 goto enable_scaling; 10026 } 10027 10028 if (pm_op == UFS_RUNTIME_PM) { 10029 if (ufshcd_can_autobkops_during_suspend(hba)) { 10030 /* 10031 * The device is idle with no requests in the queue, 10032 * allow background operations if bkops status shows 10033 * that performance might be impacted. 10034 */ 10035 ret = ufshcd_bkops_ctrl(hba); 10036 if (ret) { 10037 /* 10038 * If return err in suspend flow, IO will hang. 10039 * Trigger error handler and break suspend for 10040 * error recovery. 10041 */ 10042 ufshcd_force_error_recovery(hba); 10043 ret = -EBUSY; 10044 goto enable_scaling; 10045 } 10046 } else { 10047 /* make sure that auto bkops is disabled */ 10048 ufshcd_disable_auto_bkops(hba); 10049 } 10050 /* 10051 * If device needs to do BKOP or WB buffer flush during 10052 * Hibern8, keep device power mode as "active power mode" 10053 * and VCC supply. 10054 */ 10055 hba->dev_info.b_rpm_dev_flush_capable = 10056 hba->auto_bkops_enabled || 10057 (((req_link_state == UIC_LINK_HIBERN8_STATE) || 10058 ((req_link_state == UIC_LINK_ACTIVE_STATE) && 10059 ufshcd_is_auto_hibern8_enabled(hba))) && 10060 ufshcd_wb_need_flush(hba)); 10061 } 10062 10063 flush_work(&hba->eeh_work); 10064 cancel_delayed_work_sync(&hba->ufs_rtc_update_work); 10065 10066 ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); 10067 if (ret) 10068 goto enable_scaling; 10069 10070 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) { 10071 if (pm_op != UFS_RUNTIME_PM) 10072 /* ensure that bkops is disabled */ 10073 ufshcd_disable_auto_bkops(hba); 10074 10075 if (!hba->dev_info.b_rpm_dev_flush_capable) { 10076 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode); 10077 if (ret && pm_op != UFS_SHUTDOWN_PM) { 10078 /* 10079 * If return err in suspend flow, IO will hang. 10080 * Trigger error handler and break suspend for 10081 * error recovery. 10082 */ 10083 ufshcd_force_error_recovery(hba); 10084 ret = -EBUSY; 10085 } 10086 if (ret) 10087 goto enable_scaling; 10088 } 10089 } 10090 10091 /* 10092 * In the case of DeepSleep, the device is expected to remain powered 10093 * with the link off, so do not check for bkops. 10094 */ 10095 check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba); 10096 ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops); 10097 if (ret && pm_op != UFS_SHUTDOWN_PM) { 10098 /* 10099 * If return err in suspend flow, IO will hang. 10100 * Trigger error handler and break suspend for 10101 * error recovery. 10102 */ 10103 ufshcd_force_error_recovery(hba); 10104 ret = -EBUSY; 10105 } 10106 if (ret) 10107 goto set_dev_active; 10108 10109vops_suspend: 10110 /* 10111 * Call vendor specific suspend callback. As these callbacks may access 10112 * vendor specific host controller register space call them before the 10113 * host clocks are ON. 10114 */ 10115 ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); 10116 if (ret) 10117 goto set_link_active; 10118 10119 goto out; 10120 10121set_link_active: 10122 /* 10123 * Device hardware reset is required to exit DeepSleep. Also, for 10124 * DeepSleep, the link is off so host reset and restore will be done 10125 * further below. 10126 */ 10127 if (ufshcd_is_ufs_dev_deepsleep(hba)) { 10128 ufshcd_device_reset(hba); 10129 WARN_ON(!ufshcd_is_link_off(hba)); 10130 } 10131 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba)) 10132 ufshcd_set_link_active(hba); 10133 else if (ufshcd_is_link_off(hba)) 10134 ufshcd_host_reset_and_restore(hba); 10135set_dev_active: 10136 /* Can also get here needing to exit DeepSleep */ 10137 if (ufshcd_is_ufs_dev_deepsleep(hba)) { 10138 ufshcd_device_reset(hba); 10139 ufshcd_host_reset_and_restore(hba); 10140 } 10141 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE)) 10142 ufshcd_disable_auto_bkops(hba); 10143enable_scaling: 10144 if (ufshcd_is_clkscaling_supported(hba)) 10145 ufshcd_clk_scaling_suspend(hba, false); 10146 10147 hba->dev_info.b_rpm_dev_flush_capable = false; 10148out: 10149 if (hba->dev_info.b_rpm_dev_flush_capable) { 10150 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work, 10151 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS)); 10152 } 10153 10154 if (ret) { 10155 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret); 10156 hba->clk_gating.is_suspended = false; 10157 ufshcd_release(hba); 10158 } 10159 hba->pm_op_in_progress = false; 10160 return ret; 10161} 10162 10163#ifdef CONFIG_PM 10164static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) 10165{ 10166 int ret; 10167 enum uic_link_state old_link_state = hba->uic_link_state; 10168 10169 hba->pm_op_in_progress = true; 10170 10171 /* 10172 * Call vendor specific resume callback. As these callbacks may access 10173 * vendor specific host controller register space call them when the 10174 * host clocks are ON. 10175 */ 10176 ret = ufshcd_vops_resume(hba, pm_op); 10177 if (ret) 10178 goto out; 10179 10180 /* For DeepSleep, the only supported option is to have the link off */ 10181 WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba)); 10182 10183 if (ufshcd_is_link_hibern8(hba)) { 10184 ret = ufshcd_uic_hibern8_exit(hba); 10185 if (!ret) { 10186 ufshcd_set_link_active(hba); 10187 } else { 10188 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 10189 __func__, ret); 10190 /* 10191 * If the h8 exit fails during the runtime resume 10192 * process, it becomes stuck and cannot be recovered 10193 * through the error handler. To fix this, use link 10194 * recovery instead of the error handler. 10195 */ 10196 ret = ufshcd_link_recovery(hba); 10197 if (ret) 10198 goto vendor_suspend; 10199 } 10200 } else if (ufshcd_is_link_off(hba)) { 10201 /* 10202 * A full initialization of the host and the device is 10203 * required since the link was put to off during suspend. 10204 * Note, in the case of DeepSleep, the device will exit 10205 * DeepSleep due to device reset. 10206 */ 10207 ret = ufshcd_reset_and_restore(hba); 10208 /* 10209 * ufshcd_reset_and_restore() should have already 10210 * set the link state as active 10211 */ 10212 if (ret || !ufshcd_is_link_active(hba)) 10213 goto vendor_suspend; 10214 } 10215 10216 if (!ufshcd_is_ufs_dev_active(hba)) { 10217 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE); 10218 if (ret) 10219 goto set_old_link_state; 10220 ufshcd_set_timestamp_attr(hba); 10221 schedule_delayed_work(&hba->ufs_rtc_update_work, 10222 msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); 10223 } 10224 10225 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) 10226 ufshcd_enable_auto_bkops(hba); 10227 else 10228 /* 10229 * If BKOPs operations are urgently needed at this moment then 10230 * keep auto-bkops enabled or else disable it. 10231 */ 10232 ufshcd_bkops_ctrl(hba); 10233 10234 if (hba->ee_usr_mask) 10235 ufshcd_write_ee_control(hba); 10236 10237 if (ufshcd_is_clkscaling_supported(hba)) 10238 ufshcd_clk_scaling_suspend(hba, false); 10239 10240 if (hba->dev_info.b_rpm_dev_flush_capable) { 10241 hba->dev_info.b_rpm_dev_flush_capable = false; 10242 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work); 10243 } 10244 10245 ufshcd_configure_auto_hibern8(hba); 10246 10247 goto out; 10248 10249set_old_link_state: 10250 ufshcd_link_state_transition(hba, old_link_state, 0); 10251vendor_suspend: 10252 ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); 10253 ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); 10254out: 10255 if (ret) 10256 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret); 10257 hba->clk_gating.is_suspended = false; 10258 ufshcd_release(hba); 10259 hba->pm_op_in_progress = false; 10260 return ret; 10261} 10262 10263static int ufshcd_wl_runtime_suspend(struct device *dev) 10264{ 10265 struct scsi_device *sdev = to_scsi_device(dev); 10266 struct ufs_hba *hba; 10267 int ret; 10268 ktime_t start = ktime_get(); 10269 10270 hba = shost_priv(sdev->host); 10271 10272 ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM); 10273 if (ret) 10274 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10275 10276 trace_ufshcd_wl_runtime_suspend(hba, ret, 10277 ktime_to_us(ktime_sub(ktime_get(), start)), 10278 hba->curr_dev_pwr_mode, hba->uic_link_state); 10279 10280 return ret; 10281} 10282 10283static int ufshcd_wl_runtime_resume(struct device *dev) 10284{ 10285 struct scsi_device *sdev = to_scsi_device(dev); 10286 struct ufs_hba *hba; 10287 int ret = 0; 10288 ktime_t start = ktime_get(); 10289 10290 hba = shost_priv(sdev->host); 10291 10292 ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM); 10293 if (ret) 10294 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10295 10296 trace_ufshcd_wl_runtime_resume(hba, ret, 10297 ktime_to_us(ktime_sub(ktime_get(), start)), 10298 hba->curr_dev_pwr_mode, hba->uic_link_state); 10299 10300 return ret; 10301} 10302#endif 10303 10304#ifdef CONFIG_PM_SLEEP 10305static int ufshcd_wl_suspend(struct device *dev) 10306{ 10307 struct scsi_device *sdev = to_scsi_device(dev); 10308 struct ufs_hba *hba; 10309 int ret = 0; 10310 ktime_t start = ktime_get(); 10311 10312 hba = shost_priv(sdev->host); 10313 down(&hba->host_sem); 10314 hba->system_suspending = true; 10315 10316 if (pm_runtime_suspended(dev)) 10317 goto out; 10318 10319 ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM); 10320 if (ret) { 10321 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10322 up(&hba->host_sem); 10323 } 10324 10325out: 10326 if (!ret) 10327 hba->is_sys_suspended = true; 10328 trace_ufshcd_wl_suspend(hba, ret, 10329 ktime_to_us(ktime_sub(ktime_get(), start)), 10330 hba->curr_dev_pwr_mode, hba->uic_link_state); 10331 10332 return ret; 10333} 10334 10335static int ufshcd_wl_resume(struct device *dev) 10336{ 10337 struct scsi_device *sdev = to_scsi_device(dev); 10338 struct ufs_hba *hba; 10339 int ret = 0; 10340 ktime_t start = ktime_get(); 10341 10342 hba = shost_priv(sdev->host); 10343 10344 if (pm_runtime_suspended(dev)) 10345 goto out; 10346 10347 ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM); 10348 if (ret) 10349 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 10350out: 10351 trace_ufshcd_wl_resume(hba, ret, 10352 ktime_to_us(ktime_sub(ktime_get(), start)), 10353 hba->curr_dev_pwr_mode, hba->uic_link_state); 10354 if (!ret) 10355 hba->is_sys_suspended = false; 10356 hba->system_suspending = false; 10357 up(&hba->host_sem); 10358 return ret; 10359} 10360#endif 10361 10362/** 10363 * ufshcd_suspend - helper function for suspend operations 10364 * @hba: per adapter instance 10365 * 10366 * This function will put disable irqs, turn off clocks 10367 * and set vreg and hba-vreg in lpm mode. 10368 * 10369 * Return: 0 upon success; < 0 upon failure. 10370 */ 10371static int ufshcd_suspend(struct ufs_hba *hba) 10372{ 10373 int ret; 10374 10375 if (!hba->is_powered) 10376 return 0; 10377 /* 10378 * Disable the host irq as host controller as there won't be any 10379 * host controller transaction expected till resume. 10380 */ 10381 ufshcd_disable_irq(hba); 10382 ret = ufshcd_setup_clocks(hba, false); 10383 if (ret) { 10384 ufshcd_enable_irq(hba); 10385 goto out; 10386 } 10387 if (ufshcd_is_clkgating_allowed(hba)) { 10388 hba->clk_gating.state = CLKS_OFF; 10389 trace_ufshcd_clk_gating(hba, 10390 hba->clk_gating.state); 10391 } 10392 10393 ufshcd_vreg_set_lpm(hba); 10394 /* Put the host controller in low power mode if possible */ 10395 ufshcd_hba_vreg_set_lpm(hba); 10396 ufshcd_pm_qos_update(hba, false); 10397out: 10398 if (ret) 10399 ufshcd_update_evt_hist(hba, UFS_EVT_SUSPEND_ERR, (u32)ret); 10400 return ret; 10401} 10402 10403#ifdef CONFIG_PM 10404/** 10405 * ufshcd_resume - helper function for resume operations 10406 * @hba: per adapter instance 10407 * 10408 * This function basically turns on the regulators, clocks and 10409 * irqs of the hba. 10410 * 10411 * Return: 0 for success and non-zero for failure. 10412 */ 10413static int ufshcd_resume(struct ufs_hba *hba) 10414{ 10415 int ret; 10416 10417 if (!hba->is_powered) 10418 return 0; 10419 10420 ufshcd_hba_vreg_set_hpm(hba); 10421 ret = ufshcd_vreg_set_hpm(hba); 10422 if (ret) 10423 goto out; 10424 10425 /* Make sure clocks are enabled before accessing controller */ 10426 ret = ufshcd_setup_clocks(hba, true); 10427 if (ret) 10428 goto disable_vreg; 10429 10430 /* enable the host irq as host controller would be active soon */ 10431 ufshcd_enable_irq(hba); 10432 10433 goto out; 10434 10435disable_vreg: 10436 ufshcd_vreg_set_lpm(hba); 10437out: 10438 if (ret) 10439 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret); 10440 return ret; 10441} 10442#endif /* CONFIG_PM */ 10443 10444#ifdef CONFIG_PM_SLEEP 10445/** 10446 * ufshcd_system_suspend - system suspend callback 10447 * @dev: Device associated with the UFS controller. 10448 * 10449 * Executed before putting the system into a sleep state in which the contents 10450 * of main memory are preserved. 10451 * 10452 * Return: 0 for success and non-zero for failure. 10453 */ 10454int ufshcd_system_suspend(struct device *dev) 10455{ 10456 struct ufs_hba *hba = dev_get_drvdata(dev); 10457 int ret = 0; 10458 ktime_t start = ktime_get(); 10459 10460 if (pm_runtime_suspended(hba->dev)) 10461 goto out; 10462 10463 ret = ufshcd_suspend(hba); 10464out: 10465 trace_ufshcd_system_suspend(hba, ret, 10466 ktime_to_us(ktime_sub(ktime_get(), start)), 10467 hba->curr_dev_pwr_mode, hba->uic_link_state); 10468 return ret; 10469} 10470EXPORT_SYMBOL(ufshcd_system_suspend); 10471 10472/** 10473 * ufshcd_system_resume - system resume callback 10474 * @dev: Device associated with the UFS controller. 10475 * 10476 * Executed after waking the system up from a sleep state in which the contents 10477 * of main memory were preserved. 10478 * 10479 * Return: 0 for success and non-zero for failure. 10480 */ 10481int ufshcd_system_resume(struct device *dev) 10482{ 10483 struct ufs_hba *hba = dev_get_drvdata(dev); 10484 ktime_t start = ktime_get(); 10485 int ret = 0; 10486 10487 if (pm_runtime_suspended(hba->dev)) 10488 goto out; 10489 10490 ret = ufshcd_resume(hba); 10491 10492out: 10493 trace_ufshcd_system_resume(hba, ret, 10494 ktime_to_us(ktime_sub(ktime_get(), start)), 10495 hba->curr_dev_pwr_mode, hba->uic_link_state); 10496 10497 return ret; 10498} 10499EXPORT_SYMBOL(ufshcd_system_resume); 10500#endif /* CONFIG_PM_SLEEP */ 10501 10502#ifdef CONFIG_PM 10503/** 10504 * ufshcd_runtime_suspend - runtime suspend callback 10505 * @dev: Device associated with the UFS controller. 10506 * 10507 * Check the description of ufshcd_suspend() function for more details. 10508 * 10509 * Return: 0 for success and non-zero for failure. 10510 */ 10511int ufshcd_runtime_suspend(struct device *dev) 10512{ 10513 struct ufs_hba *hba = dev_get_drvdata(dev); 10514 int ret; 10515 ktime_t start = ktime_get(); 10516 10517 ret = ufshcd_suspend(hba); 10518 10519 trace_ufshcd_runtime_suspend(hba, ret, 10520 ktime_to_us(ktime_sub(ktime_get(), start)), 10521 hba->curr_dev_pwr_mode, hba->uic_link_state); 10522 return ret; 10523} 10524EXPORT_SYMBOL(ufshcd_runtime_suspend); 10525 10526/** 10527 * ufshcd_runtime_resume - runtime resume routine 10528 * @dev: Device associated with the UFS controller. 10529 * 10530 * This function basically brings controller 10531 * to active state. Following operations are done in this function: 10532 * 10533 * 1. Turn on all the controller related clocks 10534 * 2. Turn ON VCC rail 10535 * 10536 * Return: 0 upon success; < 0 upon failure. 10537 */ 10538int ufshcd_runtime_resume(struct device *dev) 10539{ 10540 struct ufs_hba *hba = dev_get_drvdata(dev); 10541 int ret; 10542 ktime_t start = ktime_get(); 10543 10544 ret = ufshcd_resume(hba); 10545 10546 trace_ufshcd_runtime_resume(hba, ret, 10547 ktime_to_us(ktime_sub(ktime_get(), start)), 10548 hba->curr_dev_pwr_mode, hba->uic_link_state); 10549 return ret; 10550} 10551EXPORT_SYMBOL(ufshcd_runtime_resume); 10552#endif /* CONFIG_PM */ 10553 10554static void ufshcd_wl_shutdown(struct device *dev) 10555{ 10556 struct scsi_device *sdev = to_scsi_device(dev); 10557 struct ufs_hba *hba = shost_priv(sdev->host); 10558 10559 down(&hba->host_sem); 10560 hba->shutting_down = true; 10561 up(&hba->host_sem); 10562 10563 /* Turn on everything while shutting down */ 10564 ufshcd_rpm_get_sync(hba); 10565 scsi_device_quiesce(sdev); 10566 shost_for_each_device(sdev, hba->host) { 10567 if (sdev == hba->ufs_device_wlun) 10568 continue; 10569 mutex_lock(&sdev->state_mutex); 10570 scsi_device_set_state(sdev, SDEV_OFFLINE); 10571 mutex_unlock(&sdev->state_mutex); 10572 } 10573 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); 10574 10575 /* 10576 * Next, turn off the UFS controller and the UFS regulators. Disable 10577 * clocks. 10578 */ 10579 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba)) 10580 ufshcd_suspend(hba); 10581 10582 hba->is_powered = false; 10583} 10584 10585/** 10586 * ufshcd_remove - de-allocate SCSI host and host memory space 10587 * data structure memory 10588 * @hba: per adapter instance 10589 */ 10590void ufshcd_remove(struct ufs_hba *hba) 10591{ 10592 if (hba->ufs_device_wlun) 10593 ufshcd_rpm_get_sync(hba); 10594 ufs_hwmon_remove(hba); 10595 ufs_bsg_remove(hba); 10596 ufs_rpmb_remove(hba); 10597 ufs_sysfs_remove_nodes(hba->dev); 10598 cancel_delayed_work_sync(&hba->ufs_rtc_update_work); 10599 blk_mq_destroy_queue(hba->tmf_queue); 10600 blk_put_queue(hba->tmf_queue); 10601 blk_mq_free_tag_set(&hba->tmf_tag_set); 10602 if (hba->scsi_host_added) 10603 scsi_remove_host(hba->host); 10604 /* disable interrupts */ 10605 ufshcd_disable_intr(hba, hba->intr_mask); 10606 ufshcd_hba_stop(hba); 10607 ufshcd_hba_exit(hba); 10608} 10609EXPORT_SYMBOL_GPL(ufshcd_remove); 10610 10611#ifdef CONFIG_PM_SLEEP 10612int ufshcd_system_freeze(struct device *dev) 10613{ 10614 10615 return ufshcd_system_suspend(dev); 10616 10617} 10618EXPORT_SYMBOL_GPL(ufshcd_system_freeze); 10619 10620int ufshcd_system_restore(struct device *dev) 10621{ 10622 10623 struct ufs_hba *hba = dev_get_drvdata(dev); 10624 int ret; 10625 10626 ret = ufshcd_system_resume(dev); 10627 if (ret) 10628 return ret; 10629 10630 /* Configure UTRL and UTMRL base address registers */ 10631 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 10632 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 10633 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 10634 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 10635 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 10636 REG_UTP_TASK_REQ_LIST_BASE_L); 10637 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 10638 REG_UTP_TASK_REQ_LIST_BASE_H); 10639 /* 10640 * Make sure that UTRL and UTMRL base address registers 10641 * are updated with the latest queue addresses. Only after 10642 * updating these addresses, we can queue the new commands. 10643 */ 10644 ufshcd_readl(hba, REG_UTP_TASK_REQ_LIST_BASE_H); 10645 10646 return 0; 10647 10648} 10649EXPORT_SYMBOL_GPL(ufshcd_system_restore); 10650 10651int ufshcd_system_thaw(struct device *dev) 10652{ 10653 return ufshcd_system_resume(dev); 10654} 10655EXPORT_SYMBOL_GPL(ufshcd_system_thaw); 10656#endif /* CONFIG_PM_SLEEP */ 10657 10658/** 10659 * ufshcd_set_dma_mask - Set dma mask based on the controller 10660 * addressing capability 10661 * @hba: per adapter instance 10662 * 10663 * Return: 0 for success, non-zero for failure. 10664 */ 10665static int ufshcd_set_dma_mask(struct ufs_hba *hba) 10666{ 10667 if (hba->vops && hba->vops->set_dma_mask) 10668 return hba->vops->set_dma_mask(hba); 10669 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) { 10670 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64))) 10671 return 0; 10672 } 10673 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32)); 10674} 10675 10676/** 10677 * ufshcd_devres_release - devres cleanup handler, invoked during release of 10678 * hba->dev 10679 * @host: pointer to SCSI host 10680 */ 10681static void ufshcd_devres_release(void *host) 10682{ 10683 scsi_host_put(host); 10684} 10685 10686/** 10687 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) 10688 * @dev: pointer to device handle 10689 * @hba_handle: driver private handle 10690 * 10691 * Return: 0 on success, non-zero value on failure. 10692 * 10693 * NOTE: There is no corresponding ufshcd_dealloc_host() because this function 10694 * keeps track of its allocations using devres and deallocates everything on 10695 * device removal automatically. 10696 */ 10697int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) 10698{ 10699 struct Scsi_Host *host; 10700 struct ufs_hba *hba; 10701 int err = 0; 10702 10703 if (!dev) { 10704 dev_err(dev, 10705 "Invalid memory reference for dev is NULL\n"); 10706 err = -ENODEV; 10707 goto out_error; 10708 } 10709 10710 host = scsi_host_alloc(&ufshcd_driver_template, 10711 sizeof(struct ufs_hba)); 10712 if (!host) { 10713 dev_err(dev, "scsi_host_alloc failed\n"); 10714 err = -ENOMEM; 10715 goto out_error; 10716 } 10717 10718 err = devm_add_action_or_reset(dev, ufshcd_devres_release, 10719 host); 10720 if (err) 10721 return err; 10722 10723 host->nr_maps = HCTX_TYPE_POLL + 1; 10724 hba = shost_priv(host); 10725 hba->host = host; 10726 hba->dev = dev; 10727 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL; 10728 hba->nop_out_timeout = NOP_OUT_TIMEOUT; 10729 ufshcd_set_sg_entry_size(hba, sizeof(struct ufshcd_sg_entry)); 10730 INIT_LIST_HEAD(&hba->clk_list_head); 10731 spin_lock_init(&hba->outstanding_lock); 10732 10733 *hba_handle = hba; 10734 10735out_error: 10736 return err; 10737} 10738EXPORT_SYMBOL(ufshcd_alloc_host); 10739 10740/* This function exists because blk_mq_alloc_tag_set() requires this. */ 10741static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx, 10742 const struct blk_mq_queue_data *qd) 10743{ 10744 WARN_ON_ONCE(true); 10745 return BLK_STS_NOTSUPP; 10746} 10747 10748static const struct blk_mq_ops ufshcd_tmf_ops = { 10749 .queue_rq = ufshcd_queue_tmf, 10750}; 10751 10752static int ufshcd_add_scsi_host(struct ufs_hba *hba) 10753{ 10754 int err; 10755 10756 WARN_ON_ONCE(!hba->host->can_queue); 10757 WARN_ON_ONCE(!hba->host->cmd_per_lun); 10758 10759 if (is_mcq_supported(hba)) { 10760 ufshcd_mcq_enable(hba); 10761 err = ufshcd_alloc_mcq(hba); 10762 if (err) { 10763 /* Continue with SDB mode */ 10764 ufshcd_mcq_disable(hba); 10765 use_mcq_mode = false; 10766 dev_err(hba->dev, "MCQ mode is disabled, err=%d\n", 10767 err); 10768 } 10769 } 10770 if (!is_mcq_supported(hba) && !hba->lsdb_sup) { 10771 dev_err(hba->dev, 10772 "%s: failed to initialize (legacy doorbell mode not supported)\n", 10773 __func__); 10774 return -EINVAL; 10775 } 10776 10777 err = scsi_add_host(hba->host, hba->dev); 10778 if (err) { 10779 dev_err(hba->dev, "scsi_add_host failed\n"); 10780 return err; 10781 } 10782 hba->scsi_host_added = true; 10783 10784 hba->tmf_tag_set = (struct blk_mq_tag_set) { 10785 .nr_hw_queues = 1, 10786 .queue_depth = hba->nutmrs, 10787 .ops = &ufshcd_tmf_ops, 10788 }; 10789 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set); 10790 if (err < 0) 10791 goto remove_scsi_host; 10792 hba->tmf_queue = blk_mq_alloc_queue(&hba->tmf_tag_set, NULL, NULL); 10793 if (IS_ERR(hba->tmf_queue)) { 10794 err = PTR_ERR(hba->tmf_queue); 10795 goto free_tmf_tag_set; 10796 } 10797 hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs, 10798 sizeof(*hba->tmf_rqs), GFP_KERNEL); 10799 if (!hba->tmf_rqs) { 10800 err = -ENOMEM; 10801 goto free_tmf_queue; 10802 } 10803 10804 return 0; 10805 10806free_tmf_queue: 10807 blk_mq_destroy_queue(hba->tmf_queue); 10808 blk_put_queue(hba->tmf_queue); 10809 10810free_tmf_tag_set: 10811 blk_mq_free_tag_set(&hba->tmf_tag_set); 10812 10813remove_scsi_host: 10814 if (hba->scsi_host_added) 10815 scsi_remove_host(hba->host); 10816 10817 return err; 10818} 10819 10820/** 10821 * ufshcd_init - Driver initialization routine 10822 * @hba: per-adapter instance 10823 * @mmio_base: base register address 10824 * @irq: Interrupt line of device 10825 * 10826 * Return: 0 on success; < 0 on failure. 10827 */ 10828int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) 10829{ 10830 int err; 10831 struct Scsi_Host *host = hba->host; 10832 struct device *dev = hba->dev; 10833 10834 /* 10835 * dev_set_drvdata() must be called before any callbacks are registered 10836 * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon, 10837 * sysfs). 10838 */ 10839 dev_set_drvdata(dev, hba); 10840 10841 if (!mmio_base) { 10842 dev_err(hba->dev, 10843 "Invalid memory reference for mmio_base is NULL\n"); 10844 err = -ENODEV; 10845 goto out_error; 10846 } 10847 10848 hba->mmio_base = mmio_base; 10849 hba->irq = irq; 10850 hba->vps = &ufs_hba_vps; 10851 10852 /* 10853 * Initialize clk_gating.lock early since it is being used in 10854 * ufshcd_setup_clocks() 10855 */ 10856 spin_lock_init(&hba->clk_gating.lock); 10857 10858 /* Initialize mutex for PM QoS request synchronization */ 10859 mutex_init(&hba->pm_qos_mutex); 10860 10861 /* 10862 * Set the default power management level for runtime and system PM. 10863 * Host controller drivers can override them in their 10864 * 'ufs_hba_variant_ops::init' callback. 10865 * 10866 * Default power saving mode is to keep UFS link in Hibern8 state 10867 * and UFS device in sleep state. 10868 */ 10869 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( 10870 UFS_SLEEP_PWR_MODE, 10871 UIC_LINK_HIBERN8_STATE); 10872 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( 10873 UFS_SLEEP_PWR_MODE, 10874 UIC_LINK_HIBERN8_STATE); 10875 10876 /* 10877 * Most ufs devices require 1ms delay after vcc is powered off before 10878 * it can be powered on again. Set the default to 2ms. The platform 10879 * drivers can override this setting as needed. 10880 */ 10881 hba->vcc_off_delay_us = 2000; 10882 10883 err = ufshcd_hba_init(hba); 10884 if (err) 10885 goto out_error; 10886 10887 /* Read capabilities registers */ 10888 err = ufshcd_hba_capabilities(hba); 10889 if (err) 10890 goto out_disable; 10891 10892 /* Get UFS version supported by the controller */ 10893 hba->ufs_version = ufshcd_get_ufs_version(hba); 10894 10895 /* Get Interrupt bit mask per version */ 10896 hba->intr_mask = ufshcd_get_intr_mask(hba); 10897 10898 err = ufshcd_set_dma_mask(hba); 10899 if (err) { 10900 dev_err(hba->dev, "set dma mask failed\n"); 10901 goto out_disable; 10902 } 10903 10904 /* Allocate memory for host memory space */ 10905 err = ufshcd_memory_alloc(hba); 10906 if (err) { 10907 dev_err(hba->dev, "Memory allocation failed\n"); 10908 goto out_disable; 10909 } 10910 10911 /* Configure LRB */ 10912 ufshcd_host_memory_configure(hba); 10913 10914 host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 10915 /* 10916 * Set the queue depth for WLUNs. ufs_get_device_desc() will increase 10917 * host->cmd_per_lun to a larger value. 10918 */ 10919 host->cmd_per_lun = 1; 10920 host->max_id = UFSHCD_MAX_ID; 10921 host->max_lun = UFS_MAX_LUNS; 10922 host->max_channel = UFSHCD_MAX_CHANNEL; 10923 host->unique_id = host->host_no; 10924 host->max_cmd_len = UFS_CDB_SIZE; 10925 host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING); 10926 10927 /* Use default RPM delay if host not set */ 10928 if (host->rpm_autosuspend_delay == 0) 10929 host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS; 10930 10931 hba->max_pwr_info.is_valid = false; 10932 10933 /* Initialize work queues */ 10934 hba->eh_wq = alloc_ordered_workqueue("ufs_eh_wq_%d", WQ_MEM_RECLAIM, 10935 hba->host->host_no); 10936 if (!hba->eh_wq) { 10937 dev_err(hba->dev, "%s: failed to create eh workqueue\n", 10938 __func__); 10939 err = -ENOMEM; 10940 goto out_disable; 10941 } 10942 INIT_WORK(&hba->eh_work, ufshcd_err_handler); 10943 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler); 10944 10945 sema_init(&hba->host_sem, 1); 10946 10947 /* Initialize UIC command mutex */ 10948 mutex_init(&hba->uic_cmd_mutex); 10949 10950 /* Initialize mutex for device management commands */ 10951 mutex_init(&hba->dev_cmd.lock); 10952 10953 /* Initialize mutex for exception event control */ 10954 mutex_init(&hba->ee_ctrl_mutex); 10955 10956 mutex_init(&hba->wb_mutex); 10957 10958 init_rwsem(&hba->clk_scaling_lock); 10959 10960 ufshcd_init_clk_gating(hba); 10961 10962 ufshcd_init_clk_scaling(hba); 10963 10964 /* 10965 * In order to avoid any spurious interrupt immediately after 10966 * registering UFS controller interrupt handler, clear any pending UFS 10967 * interrupt status and disable all the UFS interrupts. 10968 */ 10969 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS), 10970 REG_INTERRUPT_STATUS); 10971 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE); 10972 /* 10973 * Make sure that UFS interrupts are disabled and any pending interrupt 10974 * status is cleared before registering UFS interrupt handler. 10975 */ 10976 ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 10977 10978 /* IRQ registration */ 10979 err = devm_request_threaded_irq(dev, irq, ufshcd_intr, ufshcd_threaded_intr, 10980 IRQF_ONESHOT | IRQF_SHARED, UFSHCD, hba); 10981 if (err) { 10982 dev_err(hba->dev, "request irq failed\n"); 10983 goto out_disable; 10984 } else { 10985 hba->is_irq_enabled = true; 10986 } 10987 10988 /* Reset the attached device */ 10989 ufshcd_device_reset(hba); 10990 10991 ufshcd_init_crypto(hba); 10992 10993 /* Host controller enable */ 10994 err = ufshcd_hba_enable(hba); 10995 if (err) { 10996 dev_err(hba->dev, "Host controller enable failed\n"); 10997 ufshcd_print_evt_hist(hba); 10998 ufshcd_print_host_state(hba); 10999 goto out_disable; 11000 } 11001 11002 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work); 11003 INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work); 11004 11005 /* Set the default auto-hiberate idle timer value to 150 ms */ 11006 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) { 11007 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) | 11008 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); 11009 } 11010 11011 err = ufshcd_add_scsi_host(hba); 11012 if (err) 11013 goto out_disable; 11014 11015 /* Hold auto suspend until async scan completes */ 11016 pm_runtime_get_sync(dev); 11017 11018 /* 11019 * We are assuming that device wasn't put in sleep/power-down 11020 * state exclusively during the boot stage before kernel. 11021 * This assumption helps avoid doing link startup twice during 11022 * ufshcd_probe_hba(). 11023 */ 11024 ufshcd_set_ufs_dev_active(hba); 11025 11026 /* Initialize hba, detect and initialize UFS device */ 11027 ktime_t probe_start = ktime_get(); 11028 11029 hba->ufshcd_state = UFSHCD_STATE_RESET; 11030 11031 err = ufshcd_link_startup(hba); 11032 if (err) 11033 goto out_disable; 11034 11035 if (hba->mcq_enabled) 11036 ufshcd_config_mcq(hba); 11037 11038 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) 11039 goto initialized; 11040 11041 /* Debug counters initialization */ 11042 ufshcd_clear_dbg_ufs_stats(hba); 11043 11044 /* UniPro link is active now */ 11045 ufshcd_set_link_active(hba); 11046 11047 /* Verify device initialization by sending NOP OUT UPIU */ 11048 err = ufshcd_verify_dev_init(hba); 11049 if (err) 11050 goto out_disable; 11051 11052 /* Initiate UFS initialization, and waiting until completion */ 11053 err = ufshcd_complete_dev_init(hba); 11054 if (err) 11055 goto out_disable; 11056 11057 err = ufshcd_device_params_init(hba); 11058 if (err) 11059 goto out_disable; 11060 11061 err = ufshcd_post_device_init(hba); 11062 11063initialized: 11064 ufshcd_process_probe_result(hba, probe_start, err); 11065 if (err) 11066 goto out_disable; 11067 11068 ufs_sysfs_add_nodes(hba->dev); 11069 async_schedule(ufshcd_async_scan, hba); 11070 11071 device_enable_async_suspend(dev); 11072 ufshcd_pm_qos_init(hba); 11073 return 0; 11074 11075out_disable: 11076 hba->is_irq_enabled = false; 11077 ufshcd_hba_exit(hba); 11078out_error: 11079 return err > 0 ? -EIO : err; 11080} 11081EXPORT_SYMBOL_GPL(ufshcd_init); 11082 11083void ufshcd_resume_complete(struct device *dev) 11084{ 11085 struct ufs_hba *hba = dev_get_drvdata(dev); 11086 11087 if (hba->complete_put) { 11088 ufshcd_rpm_put(hba); 11089 hba->complete_put = false; 11090 } 11091} 11092EXPORT_SYMBOL_GPL(ufshcd_resume_complete); 11093 11094static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba) 11095{ 11096 struct device *dev = &hba->ufs_device_wlun->sdev_gendev; 11097 enum ufs_dev_pwr_mode dev_pwr_mode; 11098 enum uic_link_state link_state; 11099 unsigned long flags; 11100 bool res; 11101 11102 spin_lock_irqsave(&dev->power.lock, flags); 11103 dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl); 11104 link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl); 11105 res = pm_runtime_suspended(dev) && 11106 hba->curr_dev_pwr_mode == dev_pwr_mode && 11107 hba->uic_link_state == link_state && 11108 !hba->dev_info.b_rpm_dev_flush_capable; 11109 spin_unlock_irqrestore(&dev->power.lock, flags); 11110 11111 return res; 11112} 11113 11114int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm) 11115{ 11116 struct ufs_hba *hba = dev_get_drvdata(dev); 11117 int ret; 11118 11119 /* 11120 * SCSI assumes that runtime-pm and system-pm for scsi drivers 11121 * are same. And it doesn't wake up the device for system-suspend 11122 * if it's runtime suspended. But ufs doesn't follow that. 11123 * Refer ufshcd_resume_complete() 11124 */ 11125 if (hba->ufs_device_wlun) { 11126 /* Prevent runtime suspend */ 11127 ufshcd_rpm_get_noresume(hba); 11128 /* 11129 * Check if already runtime suspended in same state as system 11130 * suspend would be. 11131 */ 11132 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) { 11133 /* RPM state is not ok for SPM, so runtime resume */ 11134 ret = ufshcd_rpm_resume(hba); 11135 if (ret < 0 && ret != -EACCES) { 11136 ufshcd_rpm_put(hba); 11137 return ret; 11138 } 11139 } 11140 hba->complete_put = true; 11141 } 11142 return 0; 11143} 11144EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare); 11145 11146int ufshcd_suspend_prepare(struct device *dev) 11147{ 11148 return __ufshcd_suspend_prepare(dev, true); 11149} 11150EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare); 11151 11152#ifdef CONFIG_PM_SLEEP 11153static int ufshcd_wl_poweroff(struct device *dev) 11154{ 11155 struct scsi_device *sdev = to_scsi_device(dev); 11156 struct ufs_hba *hba = shost_priv(sdev->host); 11157 11158 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); 11159 return 0; 11160} 11161#endif 11162 11163static int ufshcd_wl_probe(struct device *dev) 11164{ 11165 struct scsi_device *sdev = to_scsi_device(dev); 11166 11167 if (!is_device_wlun(sdev)) 11168 return -ENODEV; 11169 11170 blk_pm_runtime_init(sdev->request_queue, dev); 11171 pm_runtime_set_autosuspend_delay(dev, 0); 11172 pm_runtime_allow(dev); 11173 11174 return 0; 11175} 11176 11177static int ufshcd_wl_remove(struct device *dev) 11178{ 11179 pm_runtime_forbid(dev); 11180 return 0; 11181} 11182 11183static const struct dev_pm_ops ufshcd_wl_pm_ops = { 11184#ifdef CONFIG_PM_SLEEP 11185 .suspend = ufshcd_wl_suspend, 11186 .resume = ufshcd_wl_resume, 11187 .freeze = ufshcd_wl_suspend, 11188 .thaw = ufshcd_wl_resume, 11189 .poweroff = ufshcd_wl_poweroff, 11190 .restore = ufshcd_wl_resume, 11191#endif 11192 SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL) 11193}; 11194 11195static void ufshcd_check_header_layout(void) 11196{ 11197 /* 11198 * gcc compilers before version 10 cannot do constant-folding for 11199 * sub-byte bitfields. Hence skip the layout checks for gcc 9 and 11200 * before. 11201 */ 11202 if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000) 11203 return; 11204 11205 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11206 .cci = 3})[0] != 3); 11207 11208 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11209 .ehs_length = 2})[1] != 2); 11210 11211 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11212 .enable_crypto = 1})[2] 11213 != 0x80); 11214 11215 BUILD_BUG_ON((((u8 *)&(struct request_desc_header){ 11216 .command_type = 5, 11217 .data_direction = 3, 11218 .interrupt = 1, 11219 })[3]) != ((5 << 4) | (3 << 1) | 1)); 11220 11221 BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ 11222 .dunl = cpu_to_le32(0xdeadbeef)})[1] != 11223 cpu_to_le32(0xdeadbeef)); 11224 11225 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11226 .ocs = 4})[8] != 4); 11227 11228 BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ 11229 .cds = 5})[9] != 5); 11230 11231 BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ 11232 .dunu = cpu_to_le32(0xbadcafe)})[3] != 11233 cpu_to_le32(0xbadcafe)); 11234 11235 BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ 11236 .iid = 0xf })[4] != 0xf0); 11237 11238 BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ 11239 .command_set_type = 0xf })[4] != 0xf); 11240} 11241 11242/* 11243 * ufs_dev_wlun_template - describes ufs device wlun 11244 * ufs-device wlun - used to send pm commands 11245 * All luns are consumers of ufs-device wlun. 11246 * 11247 * Currently, no sd driver is present for wluns. 11248 * Hence the no specific pm operations are performed. 11249 * With ufs design, SSU should be sent to ufs-device wlun. 11250 * Hence register a scsi driver for ufs wluns only. 11251 */ 11252static struct scsi_driver ufs_dev_wlun_template = { 11253 .gendrv = { 11254 .name = "ufs_device_wlun", 11255 .probe = ufshcd_wl_probe, 11256 .remove = ufshcd_wl_remove, 11257 .pm = &ufshcd_wl_pm_ops, 11258 .shutdown = ufshcd_wl_shutdown, 11259 }, 11260}; 11261 11262static int __init ufshcd_core_init(void) 11263{ 11264 int ret; 11265 11266 ufshcd_check_header_layout(); 11267 11268 ufs_debugfs_init(); 11269 11270 ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv); 11271 if (ret) 11272 ufs_debugfs_exit(); 11273 return ret; 11274} 11275 11276static void __exit ufshcd_core_exit(void) 11277{ 11278 ufs_debugfs_exit(); 11279 scsi_unregister_driver(&ufs_dev_wlun_template.gendrv); 11280} 11281 11282module_init(ufshcd_core_init); 11283module_exit(ufshcd_core_exit); 11284 11285MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>"); 11286MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>"); 11287MODULE_DESCRIPTION("Generic UFS host controller driver Core"); 11288MODULE_SOFTDEP("pre: governor_simpleondemand"); 11289MODULE_LICENSE("GPL");
~/
Read-only file preview.