FILE
renesas-nand-controller.c
File preview with syntax highlighting.
linux-6.19.14/drivers/mtd/nand/raw/renesas-nand-controller.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Evatronix/Renesas R-Car Gen3, RZ/N1D, RZ/N1S, RZ/N1L NAND controller driver 4 * 5 * Copyright (C) 2021 Schneider Electric 6 * Author: Miquel RAYNAL <miquel.raynal@bootlin.com> 7 */ 8 9#include <linux/bitfield.h> 10#include <linux/clk.h> 11#include <linux/dma-mapping.h> 12#include <linux/interrupt.h> 13#include <linux/iopoll.h> 14#include <linux/module.h> 15#include <linux/mtd/mtd.h> 16#include <linux/mtd/rawnand.h> 17#include <linux/of.h> 18#include <linux/platform_device.h> 19#include <linux/pm_runtime.h> 20#include <linux/slab.h> 21 22#define COMMAND_REG 0x00 23#define COMMAND_SEQ(x) FIELD_PREP(GENMASK(5, 0), (x)) 24#define COMMAND_SEQ_10 COMMAND_SEQ(0x2A) 25#define COMMAND_SEQ_12 COMMAND_SEQ(0x0C) 26#define COMMAND_SEQ_18 COMMAND_SEQ(0x32) 27#define COMMAND_SEQ_19 COMMAND_SEQ(0x13) 28#define COMMAND_SEQ_GEN_IN COMMAND_SEQ_18 29#define COMMAND_SEQ_GEN_OUT COMMAND_SEQ_19 30#define COMMAND_SEQ_READ_PAGE COMMAND_SEQ_10 31#define COMMAND_SEQ_WRITE_PAGE COMMAND_SEQ_12 32#define COMMAND_INPUT_SEL_AHBS 0 33#define COMMAND_INPUT_SEL_DMA BIT(6) 34#define COMMAND_FIFO_SEL 0 35#define COMMAND_DATA_SEL BIT(7) 36#define COMMAND_0(x) FIELD_PREP(GENMASK(15, 8), (x)) 37#define COMMAND_1(x) FIELD_PREP(GENMASK(23, 16), (x)) 38#define COMMAND_2(x) FIELD_PREP(GENMASK(31, 24), (x)) 39 40#define CONTROL_REG 0x04 41#define CONTROL_CHECK_RB_LINE 0 42#define CONTROL_ECC_BLOCK_SIZE(x) FIELD_PREP(GENMASK(2, 1), (x)) 43#define CONTROL_ECC_BLOCK_SIZE_256 CONTROL_ECC_BLOCK_SIZE(0) 44#define CONTROL_ECC_BLOCK_SIZE_512 CONTROL_ECC_BLOCK_SIZE(1) 45#define CONTROL_ECC_BLOCK_SIZE_1024 CONTROL_ECC_BLOCK_SIZE(2) 46#define CONTROL_INT_EN BIT(4) 47#define CONTROL_ECC_EN BIT(5) 48#define CONTROL_BLOCK_SIZE(x) FIELD_PREP(GENMASK(7, 6), (x)) 49#define CONTROL_BLOCK_SIZE_32P CONTROL_BLOCK_SIZE(0) 50#define CONTROL_BLOCK_SIZE_64P CONTROL_BLOCK_SIZE(1) 51#define CONTROL_BLOCK_SIZE_128P CONTROL_BLOCK_SIZE(2) 52#define CONTROL_BLOCK_SIZE_256P CONTROL_BLOCK_SIZE(3) 53 54#define STATUS_REG 0x8 55#define MEM_RDY(cs, reg) (FIELD_GET(GENMASK(3, 0), (reg)) & BIT(cs)) 56#define CTRL_RDY(reg) (FIELD_GET(BIT(8), (reg)) == 0) 57 58#define ECC_CTRL_REG 0x18 59#define ECC_CTRL_CAP(x) FIELD_PREP(GENMASK(2, 0), (x)) 60#define ECC_CTRL_CAP_2B ECC_CTRL_CAP(0) 61#define ECC_CTRL_CAP_4B ECC_CTRL_CAP(1) 62#define ECC_CTRL_CAP_8B ECC_CTRL_CAP(2) 63#define ECC_CTRL_CAP_16B ECC_CTRL_CAP(3) 64#define ECC_CTRL_CAP_24B ECC_CTRL_CAP(4) 65#define ECC_CTRL_CAP_32B ECC_CTRL_CAP(5) 66#define ECC_CTRL_ERR_THRESHOLD(x) FIELD_PREP(GENMASK(13, 8), (x)) 67 68#define INT_MASK_REG 0x10 69#define INT_STATUS_REG 0x14 70#define INT_CMD_END BIT(1) 71#define INT_DMA_END BIT(3) 72#define INT_MEM_RDY(cs) FIELD_PREP(GENMASK(11, 8), BIT(cs)) 73#define INT_DMA_ENDED BIT(3) 74#define MEM_IS_RDY(cs, reg) (FIELD_GET(GENMASK(11, 8), (reg)) & BIT(cs)) 75#define DMA_HAS_ENDED(reg) FIELD_GET(BIT(3), (reg)) 76 77#define ECC_OFFSET_REG 0x1C 78#define ECC_OFFSET(x) FIELD_PREP(GENMASK(15, 0), (x)) 79 80#define ECC_STAT_REG 0x20 81#define ECC_STAT_CORRECTABLE(cs, reg) (FIELD_GET(GENMASK(3, 0), (reg)) & BIT(cs)) 82#define ECC_STAT_UNCORRECTABLE(cs, reg) (FIELD_GET(GENMASK(11, 8), (reg)) & BIT(cs)) 83 84#define ADDR0_COL_REG 0x24 85#define ADDR0_COL(x) FIELD_PREP(GENMASK(15, 0), (x)) 86 87#define ADDR0_ROW_REG 0x28 88#define ADDR0_ROW(x) FIELD_PREP(GENMASK(23, 0), (x)) 89 90#define ADDR1_COL_REG 0x2C 91#define ADDR1_COL(x) FIELD_PREP(GENMASK(15, 0), (x)) 92 93#define ADDR1_ROW_REG 0x30 94#define ADDR1_ROW(x) FIELD_PREP(GENMASK(23, 0), (x)) 95 96#define FIFO_DATA_REG 0x38 97 98#define DATA_REG 0x3C 99 100#define DATA_REG_SIZE_REG 0x40 101 102#define DMA_ADDR_LOW_REG 0x64 103 104#define DMA_ADDR_HIGH_REG 0x68 105 106#define DMA_CNT_REG 0x6C 107 108#define DMA_CTRL_REG 0x70 109#define DMA_CTRL_INCREMENT_BURST_4 0 110#define DMA_CTRL_REGISTER_MANAGED_MODE 0 111#define DMA_CTRL_START BIT(7) 112 113#define MEM_CTRL_REG 0x80 114#define MEM_CTRL_CS(cs) FIELD_PREP(GENMASK(1, 0), (cs)) 115#define MEM_CTRL_DIS_WP(cs) FIELD_PREP(GENMASK(11, 8), BIT((cs))) 116 117#define DATA_SIZE_REG 0x84 118#define DATA_SIZE(x) FIELD_PREP(GENMASK(14, 0), (x)) 119 120#define TIMINGS_ASYN_REG 0x88 121#define TIMINGS_ASYN_TRWP(x) FIELD_PREP(GENMASK(3, 0), max((x), 1U) - 1) 122#define TIMINGS_ASYN_TRWH(x) FIELD_PREP(GENMASK(7, 4), max((x), 1U) - 1) 123 124#define TIM_SEQ0_REG 0x90 125#define TIM_SEQ0_TCCS(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1) 126#define TIM_SEQ0_TADL(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1) 127#define TIM_SEQ0_TRHW(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1) 128#define TIM_SEQ0_TWHR(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1) 129 130#define TIM_SEQ1_REG 0x94 131#define TIM_SEQ1_TWB(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1) 132#define TIM_SEQ1_TRR(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1) 133#define TIM_SEQ1_TWW(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1) 134 135#define TIM_GEN_SEQ0_REG 0x98 136#define TIM_GEN_SEQ0_D0(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1) 137#define TIM_GEN_SEQ0_D1(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1) 138#define TIM_GEN_SEQ0_D2(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1) 139#define TIM_GEN_SEQ0_D3(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1) 140 141#define TIM_GEN_SEQ1_REG 0x9c 142#define TIM_GEN_SEQ1_D4(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1) 143#define TIM_GEN_SEQ1_D5(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1) 144#define TIM_GEN_SEQ1_D6(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1) 145#define TIM_GEN_SEQ1_D7(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1) 146 147#define TIM_GEN_SEQ2_REG 0xA0 148#define TIM_GEN_SEQ2_D8(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1) 149#define TIM_GEN_SEQ2_D9(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1) 150#define TIM_GEN_SEQ2_D10(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1) 151#define TIM_GEN_SEQ2_D11(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1) 152 153#define FIFO_INIT_REG 0xB4 154#define FIFO_INIT BIT(0) 155 156#define FIFO_STATE_REG 0xB4 157#define FIFO_STATE_R_EMPTY(reg) FIELD_GET(BIT(0), (reg)) 158#define FIFO_STATE_W_FULL(reg) FIELD_GET(BIT(1), (reg)) 159#define FIFO_STATE_C_EMPTY(reg) FIELD_GET(BIT(2), (reg)) 160#define FIFO_STATE_R_FULL(reg) FIELD_GET(BIT(6), (reg)) 161#define FIFO_STATE_W_EMPTY(reg) FIELD_GET(BIT(7), (reg)) 162 163#define GEN_SEQ_CTRL_REG 0xB8 164#define GEN_SEQ_CMD0_EN BIT(0) 165#define GEN_SEQ_CMD1_EN BIT(1) 166#define GEN_SEQ_CMD2_EN BIT(2) 167#define GEN_SEQ_CMD3_EN BIT(3) 168#define GEN_SEQ_COL_A0(x) FIELD_PREP(GENMASK(5, 4), min((x), 2U)) 169#define GEN_SEQ_COL_A1(x) FIELD_PREP(GENMASK(7, 6), min((x), 2U)) 170#define GEN_SEQ_ROW_A0(x) FIELD_PREP(GENMASK(9, 8), min((x), 3U)) 171#define GEN_SEQ_ROW_A1(x) FIELD_PREP(GENMASK(11, 10), min((x), 3U)) 172#define GEN_SEQ_DATA_EN BIT(12) 173#define GEN_SEQ_DELAY_EN(x) FIELD_PREP(GENMASK(14, 13), (x)) 174#define GEN_SEQ_DELAY0_EN GEN_SEQ_DELAY_EN(1) 175#define GEN_SEQ_DELAY1_EN GEN_SEQ_DELAY_EN(2) 176#define GEN_SEQ_IMD_SEQ BIT(15) 177#define GEN_SEQ_COMMAND_3(x) FIELD_PREP(GENMASK(26, 16), (x)) 178 179#define DMA_TLVL_REG 0x114 180#define DMA_TLVL(x) FIELD_PREP(GENMASK(7, 0), (x)) 181#define DMA_TLVL_MAX DMA_TLVL(0xFF) 182 183#define TIM_GEN_SEQ3_REG 0x134 184#define TIM_GEN_SEQ3_D12(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1) 185 186#define ECC_CNT_REG 0x14C 187#define ECC_CNT(cs, reg) FIELD_GET(GENMASK(5, 0), (reg) >> ((cs) * 8)) 188 189#define RNANDC_CS_NUM 4 190 191#define TO_CYCLES64(ps, period_ns) ((unsigned int)DIV_ROUND_UP_ULL(div_u64(ps, 1000), \ 192 period_ns)) 193 194struct rnand_chip_sel { 195 unsigned int cs; 196}; 197 198struct rnand_chip { 199 struct nand_chip chip; 200 struct list_head node; 201 int selected_die; 202 u32 ctrl; 203 unsigned int nsels; 204 u32 control; 205 u32 ecc_ctrl; 206 u32 timings_asyn; 207 u32 tim_seq0; 208 u32 tim_seq1; 209 u32 tim_gen_seq0; 210 u32 tim_gen_seq1; 211 u32 tim_gen_seq2; 212 u32 tim_gen_seq3; 213 struct rnand_chip_sel sels[] __counted_by(nsels); 214}; 215 216struct rnandc { 217 struct nand_controller controller; 218 struct device *dev; 219 void __iomem *regs; 220 unsigned long ext_clk_rate; 221 unsigned long assigned_cs; 222 struct list_head chips; 223 struct nand_chip *selected_chip; 224 struct completion complete; 225 bool use_polling; 226 u8 *buf; 227 unsigned int buf_sz; 228}; 229 230struct rnandc_op { 231 u32 command; 232 u32 addr0_col; 233 u32 addr0_row; 234 u32 addr1_col; 235 u32 addr1_row; 236 u32 data_size; 237 u32 ecc_offset; 238 u32 gen_seq_ctrl; 239 u8 *buf; 240 bool read; 241 unsigned int len; 242}; 243 244static inline struct rnandc *to_rnandc(struct nand_controller *ctrl) 245{ 246 return container_of(ctrl, struct rnandc, controller); 247} 248 249static inline struct rnand_chip *to_rnand(struct nand_chip *chip) 250{ 251 return container_of(chip, struct rnand_chip, chip); 252} 253 254static inline unsigned int to_rnandc_cs(struct rnand_chip *nand) 255{ 256 return nand->sels[nand->selected_die].cs; 257} 258 259static void rnandc_dis_correction(struct rnandc *rnandc) 260{ 261 u32 control; 262 263 control = readl_relaxed(rnandc->regs + CONTROL_REG); 264 control &= ~CONTROL_ECC_EN; 265 writel_relaxed(control, rnandc->regs + CONTROL_REG); 266} 267 268static void rnandc_en_correction(struct rnandc *rnandc) 269{ 270 u32 control; 271 272 control = readl_relaxed(rnandc->regs + CONTROL_REG); 273 control |= CONTROL_ECC_EN; 274 writel_relaxed(control, rnandc->regs + CONTROL_REG); 275} 276 277static void rnandc_clear_status(struct rnandc *rnandc) 278{ 279 writel_relaxed(0, rnandc->regs + INT_STATUS_REG); 280 writel_relaxed(0, rnandc->regs + ECC_STAT_REG); 281 writel_relaxed(0, rnandc->regs + ECC_CNT_REG); 282} 283 284static void rnandc_dis_interrupts(struct rnandc *rnandc) 285{ 286 writel_relaxed(0, rnandc->regs + INT_MASK_REG); 287} 288 289static void rnandc_en_interrupts(struct rnandc *rnandc, u32 val) 290{ 291 if (!rnandc->use_polling) 292 writel_relaxed(val, rnandc->regs + INT_MASK_REG); 293} 294 295static void rnandc_clear_fifo(struct rnandc *rnandc) 296{ 297 writel_relaxed(FIFO_INIT, rnandc->regs + FIFO_INIT_REG); 298} 299 300static void rnandc_select_target(struct nand_chip *chip, int die_nr) 301{ 302 struct rnand_chip *rnand = to_rnand(chip); 303 struct rnandc *rnandc = to_rnandc(chip->controller); 304 unsigned int cs = rnand->sels[die_nr].cs; 305 306 if (chip == rnandc->selected_chip && die_nr == rnand->selected_die) 307 return; 308 309 rnandc_clear_status(rnandc); 310 writel_relaxed(MEM_CTRL_CS(cs) | MEM_CTRL_DIS_WP(cs), rnandc->regs + MEM_CTRL_REG); 311 writel_relaxed(rnand->control, rnandc->regs + CONTROL_REG); 312 writel_relaxed(rnand->ecc_ctrl, rnandc->regs + ECC_CTRL_REG); 313 writel_relaxed(rnand->timings_asyn, rnandc->regs + TIMINGS_ASYN_REG); 314 writel_relaxed(rnand->tim_seq0, rnandc->regs + TIM_SEQ0_REG); 315 writel_relaxed(rnand->tim_seq1, rnandc->regs + TIM_SEQ1_REG); 316 writel_relaxed(rnand->tim_gen_seq0, rnandc->regs + TIM_GEN_SEQ0_REG); 317 writel_relaxed(rnand->tim_gen_seq1, rnandc->regs + TIM_GEN_SEQ1_REG); 318 writel_relaxed(rnand->tim_gen_seq2, rnandc->regs + TIM_GEN_SEQ2_REG); 319 writel_relaxed(rnand->tim_gen_seq3, rnandc->regs + TIM_GEN_SEQ3_REG); 320 321 rnandc->selected_chip = chip; 322 rnand->selected_die = die_nr; 323} 324 325static void rnandc_trigger_op(struct rnandc *rnandc, struct rnandc_op *rop) 326{ 327 writel_relaxed(rop->addr0_col, rnandc->regs + ADDR0_COL_REG); 328 writel_relaxed(rop->addr0_row, rnandc->regs + ADDR0_ROW_REG); 329 writel_relaxed(rop->addr1_col, rnandc->regs + ADDR1_COL_REG); 330 writel_relaxed(rop->addr1_row, rnandc->regs + ADDR1_ROW_REG); 331 writel_relaxed(rop->ecc_offset, rnandc->regs + ECC_OFFSET_REG); 332 writel_relaxed(rop->gen_seq_ctrl, rnandc->regs + GEN_SEQ_CTRL_REG); 333 writel_relaxed(DATA_SIZE(rop->len), rnandc->regs + DATA_SIZE_REG); 334 writel_relaxed(rop->command, rnandc->regs + COMMAND_REG); 335} 336 337static void rnandc_trigger_dma(struct rnandc *rnandc) 338{ 339 writel_relaxed(DMA_CTRL_INCREMENT_BURST_4 | 340 DMA_CTRL_REGISTER_MANAGED_MODE | 341 DMA_CTRL_START, rnandc->regs + DMA_CTRL_REG); 342} 343 344static irqreturn_t rnandc_irq_handler(int irq, void *private) 345{ 346 struct rnandc *rnandc = private; 347 348 rnandc_dis_interrupts(rnandc); 349 complete(&rnandc->complete); 350 351 return IRQ_HANDLED; 352} 353 354static int rnandc_wait_end_of_op(struct rnandc *rnandc, 355 struct nand_chip *chip) 356{ 357 struct rnand_chip *rnand = to_rnand(chip); 358 unsigned int cs = to_rnandc_cs(rnand); 359 u32 status; 360 int ret; 361 362 ret = readl_poll_timeout(rnandc->regs + STATUS_REG, status, 363 MEM_RDY(cs, status) && CTRL_RDY(status), 364 1, 100000); 365 if (ret) 366 dev_err(rnandc->dev, "Operation timed out, status: 0x%08x\n", 367 status); 368 369 return ret; 370} 371 372static int rnandc_wait_end_of_io(struct rnandc *rnandc, 373 struct nand_chip *chip) 374{ 375 int timeout_ms = 1000; 376 int ret; 377 378 if (rnandc->use_polling) { 379 struct rnand_chip *rnand = to_rnand(chip); 380 unsigned int cs = to_rnandc_cs(rnand); 381 u32 status; 382 383 ret = readl_poll_timeout(rnandc->regs + INT_STATUS_REG, status, 384 MEM_IS_RDY(cs, status) & 385 DMA_HAS_ENDED(status), 386 0, timeout_ms * 1000); 387 } else { 388 ret = wait_for_completion_timeout(&rnandc->complete, 389 msecs_to_jiffies(timeout_ms)); 390 if (!ret) 391 ret = -ETIMEDOUT; 392 else 393 ret = 0; 394 } 395 396 return ret; 397} 398 399static int rnandc_read_page_hw_ecc(struct nand_chip *chip, u8 *buf, 400 int oob_required, int page) 401{ 402 struct rnandc *rnandc = to_rnandc(chip->controller); 403 struct mtd_info *mtd = nand_to_mtd(chip); 404 struct rnand_chip *rnand = to_rnand(chip); 405 unsigned int cs = to_rnandc_cs(rnand); 406 struct rnandc_op rop = { 407 .command = COMMAND_INPUT_SEL_DMA | COMMAND_0(NAND_CMD_READ0) | 408 COMMAND_2(NAND_CMD_READSTART) | COMMAND_FIFO_SEL | 409 COMMAND_SEQ_READ_PAGE, 410 .addr0_row = page, 411 .len = mtd->writesize, 412 .ecc_offset = ECC_OFFSET(mtd->writesize + 2), 413 }; 414 unsigned int max_bitflips = 0; 415 dma_addr_t dma_addr; 416 u32 ecc_stat; 417 int bf, ret, i; 418 419 /* Prepare controller */ 420 rnandc_select_target(chip, chip->cur_cs); 421 rnandc_clear_status(rnandc); 422 reinit_completion(&rnandc->complete); 423 rnandc_en_interrupts(rnandc, INT_DMA_ENDED); 424 rnandc_en_correction(rnandc); 425 426 /* Configure DMA */ 427 dma_addr = dma_map_single(rnandc->dev, rnandc->buf, mtd->writesize, 428 DMA_FROM_DEVICE); 429 if (dma_mapping_error(rnandc->dev, dma_addr)) 430 return -ENOMEM; 431 432 writel(dma_addr, rnandc->regs + DMA_ADDR_LOW_REG); 433 writel(mtd->writesize, rnandc->regs + DMA_CNT_REG); 434 writel(DMA_TLVL_MAX, rnandc->regs + DMA_TLVL_REG); 435 436 rnandc_trigger_op(rnandc, &rop); 437 rnandc_trigger_dma(rnandc); 438 439 ret = rnandc_wait_end_of_io(rnandc, chip); 440 dma_unmap_single(rnandc->dev, dma_addr, mtd->writesize, DMA_FROM_DEVICE); 441 rnandc_dis_correction(rnandc); 442 if (ret) { 443 dev_err(rnandc->dev, "Read page operation never ending\n"); 444 return ret; 445 } 446 447 ecc_stat = readl_relaxed(rnandc->regs + ECC_STAT_REG); 448 449 if (oob_required || ECC_STAT_UNCORRECTABLE(cs, ecc_stat)) { 450 ret = nand_change_read_column_op(chip, mtd->writesize, 451 chip->oob_poi, mtd->oobsize, 452 false); 453 if (ret) 454 return ret; 455 } 456 457 if (ECC_STAT_UNCORRECTABLE(cs, ecc_stat)) { 458 for (i = 0; i < chip->ecc.steps; i++) { 459 unsigned int off = i * chip->ecc.size; 460 unsigned int eccoff = i * chip->ecc.bytes; 461 462 bf = nand_check_erased_ecc_chunk(rnandc->buf + off, 463 chip->ecc.size, 464 chip->oob_poi + 2 + eccoff, 465 chip->ecc.bytes, 466 NULL, 0, 467 chip->ecc.strength); 468 if (bf < 0) { 469 mtd->ecc_stats.failed++; 470 } else { 471 mtd->ecc_stats.corrected += bf; 472 max_bitflips = max_t(unsigned int, max_bitflips, bf); 473 } 474 } 475 } else if (ECC_STAT_CORRECTABLE(cs, ecc_stat)) { 476 bf = ECC_CNT(cs, readl_relaxed(rnandc->regs + ECC_CNT_REG)); 477 /* 478 * The number of bitflips is an approximation given the fact 479 * that this controller does not provide per-chunk details but 480 * only gives statistics on the entire page. 481 */ 482 mtd->ecc_stats.corrected += bf; 483 } 484 485 memcpy(buf, rnandc->buf, mtd->writesize); 486 487 return 0; 488} 489 490static int rnandc_read_subpage_hw_ecc(struct nand_chip *chip, u32 req_offset, 491 u32 req_len, u8 *bufpoi, int page) 492{ 493 struct rnandc *rnandc = to_rnandc(chip->controller); 494 struct mtd_info *mtd = nand_to_mtd(chip); 495 struct rnand_chip *rnand = to_rnand(chip); 496 unsigned int cs = to_rnandc_cs(rnand); 497 unsigned int page_off = round_down(req_offset, chip->ecc.size); 498 unsigned int real_len = round_up(req_offset + req_len - page_off, 499 chip->ecc.size); 500 unsigned int start_chunk = page_off / chip->ecc.size; 501 unsigned int nchunks = real_len / chip->ecc.size; 502 unsigned int ecc_off = 2 + (start_chunk * chip->ecc.bytes); 503 struct rnandc_op rop = { 504 .command = COMMAND_INPUT_SEL_AHBS | COMMAND_0(NAND_CMD_READ0) | 505 COMMAND_2(NAND_CMD_READSTART) | COMMAND_FIFO_SEL | 506 COMMAND_SEQ_READ_PAGE, 507 .addr0_row = page, 508 .addr0_col = page_off, 509 .len = real_len, 510 .ecc_offset = ECC_OFFSET(mtd->writesize + ecc_off), 511 }; 512 unsigned int max_bitflips = 0, i; 513 u32 ecc_stat; 514 int bf, ret; 515 516 /* Prepare controller */ 517 rnandc_select_target(chip, chip->cur_cs); 518 rnandc_clear_status(rnandc); 519 rnandc_en_correction(rnandc); 520 rnandc_trigger_op(rnandc, &rop); 521 522 while (!FIFO_STATE_C_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) 523 cpu_relax(); 524 525 while (FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) 526 cpu_relax(); 527 528 ioread32_rep(rnandc->regs + FIFO_DATA_REG, bufpoi + page_off, 529 real_len / 4); 530 531 if (!FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) { 532 dev_err(rnandc->dev, "Clearing residual data in the read FIFO\n"); 533 rnandc_clear_fifo(rnandc); 534 } 535 536 ret = rnandc_wait_end_of_op(rnandc, chip); 537 rnandc_dis_correction(rnandc); 538 if (ret) { 539 dev_err(rnandc->dev, "Read subpage operation never ending\n"); 540 return ret; 541 } 542 543 ecc_stat = readl_relaxed(rnandc->regs + ECC_STAT_REG); 544 545 if (ECC_STAT_UNCORRECTABLE(cs, ecc_stat)) { 546 ret = nand_change_read_column_op(chip, mtd->writesize, 547 chip->oob_poi, mtd->oobsize, 548 false); 549 if (ret) 550 return ret; 551 552 for (i = start_chunk; i < nchunks; i++) { 553 unsigned int dataoff = i * chip->ecc.size; 554 unsigned int eccoff = 2 + (i * chip->ecc.bytes); 555 556 bf = nand_check_erased_ecc_chunk(bufpoi + dataoff, 557 chip->ecc.size, 558 chip->oob_poi + eccoff, 559 chip->ecc.bytes, 560 NULL, 0, 561 chip->ecc.strength); 562 if (bf < 0) { 563 mtd->ecc_stats.failed++; 564 } else { 565 mtd->ecc_stats.corrected += bf; 566 max_bitflips = max_t(unsigned int, max_bitflips, bf); 567 } 568 } 569 } else if (ECC_STAT_CORRECTABLE(cs, ecc_stat)) { 570 bf = ECC_CNT(cs, readl_relaxed(rnandc->regs + ECC_CNT_REG)); 571 /* 572 * The number of bitflips is an approximation given the fact 573 * that this controller does not provide per-chunk details but 574 * only gives statistics on the entire page. 575 */ 576 mtd->ecc_stats.corrected += bf; 577 } 578 579 return 0; 580} 581 582static int rnandc_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf, 583 int oob_required, int page) 584{ 585 struct rnandc *rnandc = to_rnandc(chip->controller); 586 struct mtd_info *mtd = nand_to_mtd(chip); 587 struct rnand_chip *rnand = to_rnand(chip); 588 unsigned int cs = to_rnandc_cs(rnand); 589 struct rnandc_op rop = { 590 .command = COMMAND_INPUT_SEL_DMA | COMMAND_0(NAND_CMD_SEQIN) | 591 COMMAND_1(NAND_CMD_PAGEPROG) | COMMAND_FIFO_SEL | 592 COMMAND_SEQ_WRITE_PAGE, 593 .addr0_row = page, 594 .len = mtd->writesize, 595 .ecc_offset = ECC_OFFSET(mtd->writesize + 2), 596 }; 597 dma_addr_t dma_addr; 598 int ret; 599 600 memcpy(rnandc->buf, buf, mtd->writesize); 601 602 /* Prepare controller */ 603 rnandc_select_target(chip, chip->cur_cs); 604 rnandc_clear_status(rnandc); 605 reinit_completion(&rnandc->complete); 606 rnandc_en_interrupts(rnandc, INT_MEM_RDY(cs)); 607 rnandc_en_correction(rnandc); 608 609 /* Configure DMA */ 610 dma_addr = dma_map_single(rnandc->dev, (void *)rnandc->buf, mtd->writesize, 611 DMA_TO_DEVICE); 612 if (dma_mapping_error(rnandc->dev, dma_addr)) 613 return -ENOMEM; 614 615 writel(dma_addr, rnandc->regs + DMA_ADDR_LOW_REG); 616 writel(mtd->writesize, rnandc->regs + DMA_CNT_REG); 617 writel(DMA_TLVL_MAX, rnandc->regs + DMA_TLVL_REG); 618 619 rnandc_trigger_op(rnandc, &rop); 620 rnandc_trigger_dma(rnandc); 621 622 ret = rnandc_wait_end_of_io(rnandc, chip); 623 dma_unmap_single(rnandc->dev, dma_addr, mtd->writesize, DMA_TO_DEVICE); 624 rnandc_dis_correction(rnandc); 625 if (ret) { 626 dev_err(rnandc->dev, "Write page operation never ending\n"); 627 return ret; 628 } 629 630 if (!oob_required) 631 return 0; 632 633 return nand_change_write_column_op(chip, mtd->writesize, chip->oob_poi, 634 mtd->oobsize, false); 635} 636 637static int rnandc_write_subpage_hw_ecc(struct nand_chip *chip, u32 req_offset, 638 u32 req_len, const u8 *bufpoi, 639 int oob_required, int page) 640{ 641 struct rnandc *rnandc = to_rnandc(chip->controller); 642 struct mtd_info *mtd = nand_to_mtd(chip); 643 unsigned int page_off = round_down(req_offset, chip->ecc.size); 644 unsigned int real_len = round_up(req_offset + req_len - page_off, 645 chip->ecc.size); 646 unsigned int start_chunk = page_off / chip->ecc.size; 647 unsigned int ecc_off = 2 + (start_chunk * chip->ecc.bytes); 648 struct rnandc_op rop = { 649 .command = COMMAND_INPUT_SEL_AHBS | COMMAND_0(NAND_CMD_SEQIN) | 650 COMMAND_1(NAND_CMD_PAGEPROG) | COMMAND_FIFO_SEL | 651 COMMAND_SEQ_WRITE_PAGE, 652 .addr0_row = page, 653 .addr0_col = page_off, 654 .len = real_len, 655 .ecc_offset = ECC_OFFSET(mtd->writesize + ecc_off), 656 }; 657 int ret; 658 659 /* Prepare controller */ 660 rnandc_select_target(chip, chip->cur_cs); 661 rnandc_clear_status(rnandc); 662 rnandc_en_correction(rnandc); 663 rnandc_trigger_op(rnandc, &rop); 664 665 while (FIFO_STATE_W_FULL(readl(rnandc->regs + FIFO_STATE_REG))) 666 cpu_relax(); 667 668 iowrite32_rep(rnandc->regs + FIFO_DATA_REG, bufpoi + page_off, 669 real_len / 4); 670 671 while (!FIFO_STATE_W_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) 672 cpu_relax(); 673 674 ret = rnandc_wait_end_of_op(rnandc, chip); 675 rnandc_dis_correction(rnandc); 676 if (ret) { 677 dev_err(rnandc->dev, "Write subpage operation never ending\n"); 678 return ret; 679 } 680 681 return 0; 682} 683 684/* 685 * This controller is simple enough and thus does not need to use the parser 686 * provided by the core, instead, handle every situation here. 687 */ 688static int rnandc_exec_op(struct nand_chip *chip, 689 const struct nand_operation *op, bool check_only) 690{ 691 struct rnandc *rnandc = to_rnandc(chip->controller); 692 const struct nand_op_instr *instr = NULL; 693 struct rnandc_op rop = { 694 .command = COMMAND_INPUT_SEL_AHBS, 695 .gen_seq_ctrl = GEN_SEQ_IMD_SEQ, 696 }; 697 unsigned int cmd_phase = 0, addr_phase = 0, data_phase = 0, 698 delay_phase = 0, delays = 0; 699 unsigned int op_id, col_addrs, row_addrs, naddrs, remainder, words, i; 700 const u8 *addrs; 701 u32 last_bytes; 702 int ret; 703 704 if (!check_only) 705 rnandc_select_target(chip, op->cs); 706 707 for (op_id = 0; op_id < op->ninstrs; op_id++) { 708 instr = &op->instrs[op_id]; 709 710 nand_op_trace(" ", instr); 711 712 switch (instr->type) { 713 case NAND_OP_CMD_INSTR: 714 switch (cmd_phase++) { 715 case 0: 716 rop.command |= COMMAND_0(instr->ctx.cmd.opcode); 717 rop.gen_seq_ctrl |= GEN_SEQ_CMD0_EN; 718 break; 719 case 1: 720 rop.gen_seq_ctrl |= GEN_SEQ_COMMAND_3(instr->ctx.cmd.opcode); 721 rop.gen_seq_ctrl |= GEN_SEQ_CMD3_EN; 722 if (addr_phase == 0) 723 addr_phase = 1; 724 break; 725 case 2: 726 rop.command |= COMMAND_2(instr->ctx.cmd.opcode); 727 rop.gen_seq_ctrl |= GEN_SEQ_CMD2_EN; 728 if (addr_phase <= 1) 729 addr_phase = 2; 730 break; 731 case 3: 732 rop.command |= COMMAND_1(instr->ctx.cmd.opcode); 733 rop.gen_seq_ctrl |= GEN_SEQ_CMD1_EN; 734 if (addr_phase <= 1) 735 addr_phase = 2; 736 if (delay_phase == 0) 737 delay_phase = 1; 738 if (data_phase == 0) 739 data_phase = 1; 740 break; 741 default: 742 return -EOPNOTSUPP; 743 } 744 break; 745 746 case NAND_OP_ADDR_INSTR: 747 addrs = instr->ctx.addr.addrs; 748 naddrs = instr->ctx.addr.naddrs; 749 if (naddrs > 5) 750 return -EOPNOTSUPP; 751 752 col_addrs = min(2U, naddrs); 753 row_addrs = naddrs > 2 ? naddrs - col_addrs : 0; 754 755 switch (addr_phase++) { 756 case 0: 757 for (i = 0; i < col_addrs; i++) 758 rop.addr0_col |= addrs[i] << (i * 8); 759 rop.gen_seq_ctrl |= GEN_SEQ_COL_A0(col_addrs); 760 761 for (i = 0; i < row_addrs; i++) 762 rop.addr0_row |= addrs[2 + i] << (i * 8); 763 rop.gen_seq_ctrl |= GEN_SEQ_ROW_A0(row_addrs); 764 765 if (cmd_phase == 0) 766 cmd_phase = 1; 767 break; 768 case 1: 769 for (i = 0; i < col_addrs; i++) 770 rop.addr1_col |= addrs[i] << (i * 8); 771 rop.gen_seq_ctrl |= GEN_SEQ_COL_A1(col_addrs); 772 773 for (i = 0; i < row_addrs; i++) 774 rop.addr1_row |= addrs[2 + i] << (i * 8); 775 rop.gen_seq_ctrl |= GEN_SEQ_ROW_A1(row_addrs); 776 777 if (cmd_phase <= 1) 778 cmd_phase = 2; 779 break; 780 default: 781 return -EOPNOTSUPP; 782 } 783 break; 784 785 case NAND_OP_DATA_IN_INSTR: 786 rop.read = true; 787 fallthrough; 788 case NAND_OP_DATA_OUT_INSTR: 789 rop.gen_seq_ctrl |= GEN_SEQ_DATA_EN; 790 rop.buf = instr->ctx.data.buf.in; 791 rop.len = instr->ctx.data.len; 792 rop.command |= COMMAND_FIFO_SEL; 793 794 switch (data_phase++) { 795 case 0: 796 if (cmd_phase <= 2) 797 cmd_phase = 3; 798 if (addr_phase <= 1) 799 addr_phase = 2; 800 if (delay_phase == 0) 801 delay_phase = 1; 802 break; 803 default: 804 return -EOPNOTSUPP; 805 } 806 break; 807 808 case NAND_OP_WAITRDY_INSTR: 809 switch (delay_phase++) { 810 case 0: 811 rop.gen_seq_ctrl |= GEN_SEQ_DELAY0_EN; 812 813 if (cmd_phase <= 2) 814 cmd_phase = 3; 815 break; 816 case 1: 817 rop.gen_seq_ctrl |= GEN_SEQ_DELAY1_EN; 818 819 if (cmd_phase <= 3) 820 cmd_phase = 4; 821 if (data_phase == 0) 822 data_phase = 1; 823 break; 824 default: 825 return -EOPNOTSUPP; 826 } 827 break; 828 } 829 } 830 831 /* 832 * Sequence 19 is generic and dedicated to write operations. 833 * Sequence 18 is also generic and works for all other operations. 834 */ 835 if (rop.buf && !rop.read) 836 rop.command |= COMMAND_SEQ_GEN_OUT; 837 else 838 rop.command |= COMMAND_SEQ_GEN_IN; 839 840 if (delays > 1) { 841 dev_err(rnandc->dev, "Cannot handle more than one wait delay\n"); 842 return -EOPNOTSUPP; 843 } 844 845 if (check_only) 846 return 0; 847 848 rnandc_trigger_op(rnandc, &rop); 849 850 words = rop.len / sizeof(u32); 851 remainder = rop.len % sizeof(u32); 852 if (rop.buf && rop.read) { 853 while (!FIFO_STATE_C_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) 854 cpu_relax(); 855 856 while (FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) 857 cpu_relax(); 858 859 ioread32_rep(rnandc->regs + FIFO_DATA_REG, rop.buf, words); 860 if (remainder) { 861 last_bytes = readl_relaxed(rnandc->regs + FIFO_DATA_REG); 862 memcpy(rop.buf + (words * sizeof(u32)), &last_bytes, 863 remainder); 864 } 865 866 if (!FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) { 867 dev_warn(rnandc->dev, 868 "Clearing residual data in the read FIFO\n"); 869 rnandc_clear_fifo(rnandc); 870 } 871 } else if (rop.len && !rop.read) { 872 while (FIFO_STATE_W_FULL(readl(rnandc->regs + FIFO_STATE_REG))) 873 cpu_relax(); 874 875 iowrite32_rep(rnandc->regs + FIFO_DATA_REG, rop.buf, 876 DIV_ROUND_UP(rop.len, 4)); 877 878 if (remainder) { 879 last_bytes = 0; 880 memcpy(&last_bytes, rop.buf + (words * sizeof(u32)), remainder); 881 writel_relaxed(last_bytes, rnandc->regs + FIFO_DATA_REG); 882 } 883 884 while (!FIFO_STATE_W_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) 885 cpu_relax(); 886 } 887 888 ret = rnandc_wait_end_of_op(rnandc, chip); 889 if (ret) 890 return ret; 891 892 return 0; 893} 894 895static int rnandc_setup_interface(struct nand_chip *chip, int chipnr, 896 const struct nand_interface_config *conf) 897{ 898 struct rnand_chip *rnand = to_rnand(chip); 899 struct rnandc *rnandc = to_rnandc(chip->controller); 900 unsigned int period_ns = 1000000000 / rnandc->ext_clk_rate; 901 const struct nand_sdr_timings *sdr; 902 unsigned int cyc, cle, ale, bef_dly, ca_to_data; 903 904 sdr = nand_get_sdr_timings(conf); 905 if (IS_ERR(sdr)) 906 return PTR_ERR(sdr); 907 908 if (sdr->tRP_min != sdr->tWP_min || sdr->tREH_min != sdr->tWH_min) { 909 dev_err(rnandc->dev, "Read and write hold times must be identical\n"); 910 return -EINVAL; 911 } 912 913 if (chipnr < 0) 914 return 0; 915 916 rnand->timings_asyn = 917 TIMINGS_ASYN_TRWP(TO_CYCLES64(sdr->tRP_min, period_ns)) | 918 TIMINGS_ASYN_TRWH(TO_CYCLES64(sdr->tREH_min, period_ns)); 919 rnand->tim_seq0 = 920 TIM_SEQ0_TCCS(TO_CYCLES64(sdr->tCCS_min, period_ns)) | 921 TIM_SEQ0_TADL(TO_CYCLES64(sdr->tADL_min, period_ns)) | 922 TIM_SEQ0_TRHW(TO_CYCLES64(sdr->tRHW_min, period_ns)) | 923 TIM_SEQ0_TWHR(TO_CYCLES64(sdr->tWHR_min, period_ns)); 924 rnand->tim_seq1 = 925 TIM_SEQ1_TWB(TO_CYCLES64(sdr->tWB_max, period_ns)) | 926 TIM_SEQ1_TRR(TO_CYCLES64(sdr->tRR_min, period_ns)) | 927 TIM_SEQ1_TWW(TO_CYCLES64(sdr->tWW_min, period_ns)); 928 929 cyc = sdr->tDS_min + sdr->tDH_min; 930 cle = sdr->tCLH_min + sdr->tCLS_min; 931 ale = sdr->tALH_min + sdr->tALS_min; 932 bef_dly = sdr->tWB_max - sdr->tDH_min; 933 ca_to_data = sdr->tWHR_min + sdr->tREA_max - sdr->tDH_min; 934 935 /* 936 * D0 = CMD -> ADDR = tCLH + tCLS - 1 cycle 937 * D1 = CMD -> CMD = tCLH + tCLS - 1 cycle 938 * D2 = CMD -> DLY = tWB - tDH 939 * D3 = CMD -> DATA = tWHR + tREA - tDH 940 */ 941 rnand->tim_gen_seq0 = 942 TIM_GEN_SEQ0_D0(TO_CYCLES64(cle - cyc, period_ns)) | 943 TIM_GEN_SEQ0_D1(TO_CYCLES64(cle - cyc, period_ns)) | 944 TIM_GEN_SEQ0_D2(TO_CYCLES64(bef_dly, period_ns)) | 945 TIM_GEN_SEQ0_D3(TO_CYCLES64(ca_to_data, period_ns)); 946 947 /* 948 * D4 = ADDR -> CMD = tALH + tALS - 1 cyle 949 * D5 = ADDR -> ADDR = tALH + tALS - 1 cyle 950 * D6 = ADDR -> DLY = tWB - tDH 951 * D7 = ADDR -> DATA = tWHR + tREA - tDH 952 */ 953 rnand->tim_gen_seq1 = 954 TIM_GEN_SEQ1_D4(TO_CYCLES64(ale - cyc, period_ns)) | 955 TIM_GEN_SEQ1_D5(TO_CYCLES64(ale - cyc, period_ns)) | 956 TIM_GEN_SEQ1_D6(TO_CYCLES64(bef_dly, period_ns)) | 957 TIM_GEN_SEQ1_D7(TO_CYCLES64(ca_to_data, period_ns)); 958 959 /* 960 * D8 = DLY -> DATA = tRR + tREA 961 * D9 = DLY -> CMD = tRR 962 * D10 = DATA -> CMD = tCLH + tCLS - 1 cycle 963 * D11 = DATA -> DLY = tWB - tDH 964 */ 965 rnand->tim_gen_seq2 = 966 TIM_GEN_SEQ2_D8(TO_CYCLES64(sdr->tRR_min + sdr->tREA_max, period_ns)) | 967 TIM_GEN_SEQ2_D9(TO_CYCLES64(sdr->tRR_min, period_ns)) | 968 TIM_GEN_SEQ2_D10(TO_CYCLES64(cle - cyc, period_ns)) | 969 TIM_GEN_SEQ2_D11(TO_CYCLES64(bef_dly, period_ns)); 970 971 /* D12 = DATA -> END = tCLH - tDH */ 972 rnand->tim_gen_seq3 = 973 TIM_GEN_SEQ3_D12(TO_CYCLES64(sdr->tCLH_min - sdr->tDH_min, period_ns)); 974 975 return 0; 976} 977 978static int rnandc_ooblayout_ecc(struct mtd_info *mtd, int section, 979 struct mtd_oob_region *oobregion) 980{ 981 struct nand_chip *chip = mtd_to_nand(mtd); 982 unsigned int eccbytes = round_up(chip->ecc.bytes, 4) * chip->ecc.steps; 983 984 if (section) 985 return -ERANGE; 986 987 oobregion->offset = 2; 988 oobregion->length = eccbytes; 989 990 return 0; 991} 992 993static int rnandc_ooblayout_free(struct mtd_info *mtd, int section, 994 struct mtd_oob_region *oobregion) 995{ 996 struct nand_chip *chip = mtd_to_nand(mtd); 997 unsigned int eccbytes = round_up(chip->ecc.bytes, 4) * chip->ecc.steps; 998 999 if (section) 1000 return -ERANGE; 1001 1002 oobregion->offset = 2 + eccbytes; 1003 oobregion->length = mtd->oobsize - oobregion->offset; 1004 1005 return 0; 1006} 1007 1008static const struct mtd_ooblayout_ops rnandc_ooblayout_ops = { 1009 .ecc = rnandc_ooblayout_ecc, 1010 .free = rnandc_ooblayout_free, 1011}; 1012 1013static int rnandc_hw_ecc_controller_init(struct nand_chip *chip) 1014{ 1015 struct rnand_chip *rnand = to_rnand(chip); 1016 struct mtd_info *mtd = nand_to_mtd(chip); 1017 struct rnandc *rnandc = to_rnandc(chip->controller); 1018 1019 if (mtd->writesize > SZ_16K) { 1020 dev_err(rnandc->dev, "Unsupported page size\n"); 1021 return -EINVAL; 1022 } 1023 1024 switch (chip->ecc.size) { 1025 case SZ_256: 1026 rnand->control |= CONTROL_ECC_BLOCK_SIZE_256; 1027 break; 1028 case SZ_512: 1029 rnand->control |= CONTROL_ECC_BLOCK_SIZE_512; 1030 break; 1031 case SZ_1K: 1032 rnand->control |= CONTROL_ECC_BLOCK_SIZE_1024; 1033 break; 1034 default: 1035 dev_err(rnandc->dev, "Unsupported ECC chunk size\n"); 1036 return -EINVAL; 1037 } 1038 1039 switch (chip->ecc.strength) { 1040 case 2: 1041 chip->ecc.bytes = 4; 1042 rnand->ecc_ctrl |= ECC_CTRL_CAP_2B; 1043 break; 1044 case 4: 1045 chip->ecc.bytes = 7; 1046 rnand->ecc_ctrl |= ECC_CTRL_CAP_4B; 1047 break; 1048 case 8: 1049 chip->ecc.bytes = 14; 1050 rnand->ecc_ctrl |= ECC_CTRL_CAP_8B; 1051 break; 1052 case 16: 1053 chip->ecc.bytes = 28; 1054 rnand->ecc_ctrl |= ECC_CTRL_CAP_16B; 1055 break; 1056 case 24: 1057 chip->ecc.bytes = 42; 1058 rnand->ecc_ctrl |= ECC_CTRL_CAP_24B; 1059 break; 1060 case 32: 1061 chip->ecc.bytes = 56; 1062 rnand->ecc_ctrl |= ECC_CTRL_CAP_32B; 1063 break; 1064 default: 1065 dev_err(rnandc->dev, "Unsupported ECC strength\n"); 1066 return -EINVAL; 1067 } 1068 1069 rnand->ecc_ctrl |= ECC_CTRL_ERR_THRESHOLD(chip->ecc.strength); 1070 1071 mtd_set_ooblayout(mtd, &rnandc_ooblayout_ops); 1072 chip->ecc.steps = mtd->writesize / chip->ecc.size; 1073 chip->ecc.read_page = rnandc_read_page_hw_ecc; 1074 chip->ecc.read_subpage = rnandc_read_subpage_hw_ecc; 1075 chip->ecc.write_page = rnandc_write_page_hw_ecc; 1076 chip->ecc.write_subpage = rnandc_write_subpage_hw_ecc; 1077 1078 return 0; 1079} 1080 1081static int rnandc_ecc_init(struct nand_chip *chip) 1082{ 1083 struct nand_ecc_ctrl *ecc = &chip->ecc; 1084 const struct nand_ecc_props *requirements = 1085 nanddev_get_ecc_requirements(&chip->base); 1086 struct rnandc *rnandc = to_rnandc(chip->controller); 1087 int ret; 1088 1089 if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_NONE && 1090 (!ecc->size || !ecc->strength)) { 1091 if (requirements->step_size && requirements->strength) { 1092 ecc->size = requirements->step_size; 1093 ecc->strength = requirements->strength; 1094 } else { 1095 dev_err(rnandc->dev, "No minimum ECC strength\n"); 1096 return -EINVAL; 1097 } 1098 } 1099 1100 switch (ecc->engine_type) { 1101 case NAND_ECC_ENGINE_TYPE_ON_HOST: 1102 ret = rnandc_hw_ecc_controller_init(chip); 1103 if (ret) 1104 return ret; 1105 break; 1106 case NAND_ECC_ENGINE_TYPE_NONE: 1107 case NAND_ECC_ENGINE_TYPE_SOFT: 1108 case NAND_ECC_ENGINE_TYPE_ON_DIE: 1109 break; 1110 default: 1111 return -EINVAL; 1112 } 1113 1114 return 0; 1115} 1116 1117static int rnandc_attach_chip(struct nand_chip *chip) 1118{ 1119 struct rnand_chip *rnand = to_rnand(chip); 1120 struct rnandc *rnandc = to_rnandc(chip->controller); 1121 struct mtd_info *mtd = nand_to_mtd(chip); 1122 struct nand_memory_organization *memorg = nanddev_get_memorg(&chip->base); 1123 int ret; 1124 1125 /* Do not store BBT bits in the OOB section as it is not protected */ 1126 if (chip->bbt_options & NAND_BBT_USE_FLASH) 1127 chip->bbt_options |= NAND_BBT_NO_OOB; 1128 1129 if (mtd->writesize <= 512) { 1130 dev_err(rnandc->dev, "Small page devices not supported\n"); 1131 return -EINVAL; 1132 } 1133 1134 rnand->control |= CONTROL_CHECK_RB_LINE | CONTROL_INT_EN; 1135 1136 switch (memorg->pages_per_eraseblock) { 1137 case 32: 1138 rnand->control |= CONTROL_BLOCK_SIZE_32P; 1139 break; 1140 case 64: 1141 rnand->control |= CONTROL_BLOCK_SIZE_64P; 1142 break; 1143 case 128: 1144 rnand->control |= CONTROL_BLOCK_SIZE_128P; 1145 break; 1146 case 256: 1147 rnand->control |= CONTROL_BLOCK_SIZE_256P; 1148 break; 1149 default: 1150 dev_err(rnandc->dev, "Unsupported memory organization\n"); 1151 return -EINVAL; 1152 } 1153 1154 chip->options |= NAND_SUBPAGE_READ; 1155 1156 ret = rnandc_ecc_init(chip); 1157 if (ret) { 1158 dev_err(rnandc->dev, "ECC initialization failed (%d)\n", ret); 1159 return ret; 1160 } 1161 1162 /* Force an update of the configuration registers */ 1163 rnand->selected_die = -1; 1164 1165 return 0; 1166} 1167 1168static const struct nand_controller_ops rnandc_ops = { 1169 .attach_chip = rnandc_attach_chip, 1170 .exec_op = rnandc_exec_op, 1171 .setup_interface = rnandc_setup_interface, 1172}; 1173 1174static int rnandc_alloc_dma_buf(struct rnandc *rnandc, 1175 struct mtd_info *new_mtd) 1176{ 1177 unsigned int max_len = new_mtd->writesize + new_mtd->oobsize; 1178 struct rnand_chip *entry, *temp; 1179 struct nand_chip *chip; 1180 struct mtd_info *mtd; 1181 1182 list_for_each_entry_safe(entry, temp, &rnandc->chips, node) { 1183 chip = &entry->chip; 1184 mtd = nand_to_mtd(chip); 1185 max_len = max(max_len, mtd->writesize + mtd->oobsize); 1186 } 1187 1188 if (rnandc->buf && rnandc->buf_sz < max_len) { 1189 devm_kfree(rnandc->dev, rnandc->buf); 1190 rnandc->buf = NULL; 1191 } 1192 1193 if (!rnandc->buf) { 1194 rnandc->buf_sz = max_len; 1195 rnandc->buf = devm_kmalloc(rnandc->dev, max_len, 1196 GFP_KERNEL | GFP_DMA); 1197 if (!rnandc->buf) 1198 return -ENOMEM; 1199 } 1200 1201 return 0; 1202} 1203 1204static int rnandc_chip_init(struct rnandc *rnandc, struct device_node *np) 1205{ 1206 struct rnand_chip *rnand; 1207 struct mtd_info *mtd; 1208 struct nand_chip *chip; 1209 int nsels, ret, i; 1210 u32 cs; 1211 1212 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32)); 1213 if (nsels <= 0) { 1214 ret = (nsels < 0) ? nsels : -EINVAL; 1215 dev_err(rnandc->dev, "Invalid reg property (%d)\n", ret); 1216 return ret; 1217 } 1218 1219 /* Alloc the driver's NAND chip structure */ 1220 rnand = devm_kzalloc(rnandc->dev, struct_size(rnand, sels, nsels), 1221 GFP_KERNEL); 1222 if (!rnand) 1223 return -ENOMEM; 1224 1225 rnand->nsels = nsels; 1226 rnand->selected_die = -1; 1227 1228 for (i = 0; i < nsels; i++) { 1229 ret = of_property_read_u32_index(np, "reg", i, &cs); 1230 if (ret) { 1231 dev_err(rnandc->dev, "Incomplete reg property (%d)\n", ret); 1232 return ret; 1233 } 1234 1235 if (cs >= RNANDC_CS_NUM) { 1236 dev_err(rnandc->dev, "Invalid reg property (%d)\n", cs); 1237 return -EINVAL; 1238 } 1239 1240 if (test_and_set_bit(cs, &rnandc->assigned_cs)) { 1241 dev_err(rnandc->dev, "CS %d already assigned\n", cs); 1242 return -EINVAL; 1243 } 1244 1245 /* 1246 * No need to check for RB or WP properties, there is a 1:1 1247 * mandatory mapping with the CS. 1248 */ 1249 rnand->sels[i].cs = cs; 1250 } 1251 1252 chip = &rnand->chip; 1253 chip->controller = &rnandc->controller; 1254 nand_set_flash_node(chip, np); 1255 1256 mtd = nand_to_mtd(chip); 1257 mtd->dev.parent = rnandc->dev; 1258 if (!mtd->name) { 1259 dev_err(rnandc->dev, "Missing MTD label\n"); 1260 return -EINVAL; 1261 } 1262 1263 ret = nand_scan(chip, rnand->nsels); 1264 if (ret) { 1265 dev_err(rnandc->dev, "Failed to scan the NAND chip (%d)\n", ret); 1266 return ret; 1267 } 1268 1269 ret = rnandc_alloc_dma_buf(rnandc, mtd); 1270 if (ret) 1271 goto cleanup_nand; 1272 1273 ret = mtd_device_register(mtd, NULL, 0); 1274 if (ret) { 1275 dev_err(rnandc->dev, "Failed to register MTD device (%d)\n", ret); 1276 goto cleanup_nand; 1277 } 1278 1279 list_add_tail(&rnand->node, &rnandc->chips); 1280 1281 return 0; 1282 1283cleanup_nand: 1284 nand_cleanup(chip); 1285 1286 return ret; 1287} 1288 1289static void rnandc_chips_cleanup(struct rnandc *rnandc) 1290{ 1291 struct rnand_chip *entry, *temp; 1292 struct nand_chip *chip; 1293 int ret; 1294 1295 list_for_each_entry_safe(entry, temp, &rnandc->chips, node) { 1296 chip = &entry->chip; 1297 ret = mtd_device_unregister(nand_to_mtd(chip)); 1298 WARN_ON(ret); 1299 nand_cleanup(chip); 1300 list_del(&entry->node); 1301 } 1302} 1303 1304static int rnandc_chips_init(struct rnandc *rnandc) 1305{ 1306 int ret; 1307 1308 for_each_child_of_node_scoped(rnandc->dev->of_node, np) { 1309 ret = rnandc_chip_init(rnandc, np); 1310 if (ret) { 1311 rnandc_chips_cleanup(rnandc); 1312 return ret; 1313 } 1314 } 1315 1316 return 0; 1317} 1318 1319static int rnandc_probe(struct platform_device *pdev) 1320{ 1321 struct rnandc *rnandc; 1322 struct clk *eclk; 1323 int irq, ret; 1324 1325 rnandc = devm_kzalloc(&pdev->dev, sizeof(*rnandc), GFP_KERNEL); 1326 if (!rnandc) 1327 return -ENOMEM; 1328 1329 rnandc->dev = &pdev->dev; 1330 nand_controller_init(&rnandc->controller); 1331 rnandc->controller.ops = &rnandc_ops; 1332 INIT_LIST_HEAD(&rnandc->chips); 1333 init_completion(&rnandc->complete); 1334 1335 rnandc->regs = devm_platform_ioremap_resource(pdev, 0); 1336 if (IS_ERR(rnandc->regs)) 1337 return PTR_ERR(rnandc->regs); 1338 1339 ret = devm_pm_runtime_enable(&pdev->dev); 1340 if (ret) 1341 return ret; 1342 1343 ret = pm_runtime_resume_and_get(&pdev->dev); 1344 if (ret < 0) 1345 return ret; 1346 1347 /* The external NAND bus clock rate is needed for computing timings */ 1348 eclk = clk_get(&pdev->dev, "eclk"); 1349 if (IS_ERR(eclk)) { 1350 ret = PTR_ERR(eclk); 1351 goto dis_runtime_pm; 1352 } 1353 1354 rnandc->ext_clk_rate = clk_get_rate(eclk); 1355 clk_put(eclk); 1356 1357 rnandc_dis_interrupts(rnandc); 1358 irq = platform_get_irq_optional(pdev, 0); 1359 if (irq == -EPROBE_DEFER) { 1360 ret = irq; 1361 goto dis_runtime_pm; 1362 } else if (irq < 0) { 1363 dev_info(&pdev->dev, "No IRQ found, fallback to polling\n"); 1364 rnandc->use_polling = true; 1365 } else { 1366 ret = devm_request_irq(&pdev->dev, irq, rnandc_irq_handler, 0, 1367 "renesas-nand-controller", rnandc); 1368 if (ret < 0) 1369 goto dis_runtime_pm; 1370 } 1371 1372 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 1373 if (ret) 1374 goto dis_runtime_pm; 1375 1376 rnandc_clear_fifo(rnandc); 1377 1378 platform_set_drvdata(pdev, rnandc); 1379 1380 ret = rnandc_chips_init(rnandc); 1381 if (ret) 1382 goto dis_runtime_pm; 1383 1384 return 0; 1385 1386dis_runtime_pm: 1387 pm_runtime_put(&pdev->dev); 1388 1389 return ret; 1390} 1391 1392static void rnandc_remove(struct platform_device *pdev) 1393{ 1394 struct rnandc *rnandc = platform_get_drvdata(pdev); 1395 1396 rnandc_chips_cleanup(rnandc); 1397 1398 pm_runtime_put(&pdev->dev); 1399} 1400 1401static const struct of_device_id rnandc_id_table[] = { 1402 { .compatible = "renesas,rcar-gen3-nandc" }, 1403 { .compatible = "renesas,rzn1-nandc" }, 1404 {} /* sentinel */ 1405}; 1406MODULE_DEVICE_TABLE(of, rnandc_id_table); 1407 1408static struct platform_driver rnandc_driver = { 1409 .driver = { 1410 .name = "renesas-nandc", 1411 .of_match_table = rnandc_id_table, 1412 }, 1413 .probe = rnandc_probe, 1414 .remove = rnandc_remove, 1415}; 1416module_platform_driver(rnandc_driver); 1417 1418MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>"); 1419MODULE_DESCRIPTION("Renesas R-Car Gen3 & RZ/N1 NAND controller driver"); 1420MODULE_LICENSE("GPL v2");
~/
Read-only file preview.