FILE
gpmi-nand.c
File preview with syntax highlighting.
linux-6.19.14/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
1// SPDX-License-Identifier: GPL-2.0+ 2/* 3 * Freescale GPMI NAND Flash Driver 4 * 5 * Copyright (C) 2010-2015 Freescale Semiconductor, Inc. 6 * Copyright (C) 2008 Embedded Alley Solutions, Inc. 7 */ 8#include <linux/clk.h> 9#include <linux/delay.h> 10#include <linux/slab.h> 11#include <linux/sched/task_stack.h> 12#include <linux/interrupt.h> 13#include <linux/module.h> 14#include <linux/mtd/partitions.h> 15#include <linux/of.h> 16#include <linux/platform_device.h> 17#include <linux/pm_runtime.h> 18#include <linux/pinctrl/consumer.h> 19#include <linux/dma/mxs-dma.h> 20#include <linux/string_choices.h> 21#include "gpmi-nand.h" 22#include "gpmi-regs.h" 23#include "bch-regs.h" 24 25/* Resource names for the GPMI NAND driver. */ 26#define GPMI_NAND_GPMI_REGS_ADDR_RES_NAME "gpmi-nand" 27#define GPMI_NAND_BCH_REGS_ADDR_RES_NAME "bch" 28#define GPMI_NAND_BCH_INTERRUPT_RES_NAME "bch" 29 30/* Converts time to clock cycles */ 31#define TO_CYCLES(duration, period) DIV_ROUND_UP_ULL(duration, period) 32 33#define MXS_SET_ADDR 0x4 34#define MXS_CLR_ADDR 0x8 35/* 36 * Clear the bit and poll it cleared. This is usually called with 37 * a reset address and mask being either SFTRST(bit 31) or CLKGATE 38 * (bit 30). 39 */ 40static int clear_poll_bit(void __iomem *addr, u32 mask) 41{ 42 int timeout = 0x400; 43 44 /* clear the bit */ 45 writel(mask, addr + MXS_CLR_ADDR); 46 47 /* 48 * SFTRST needs 3 GPMI clocks to settle, the reference manual 49 * recommends to wait 1us. 50 */ 51 udelay(1); 52 53 /* poll the bit becoming clear */ 54 while ((readl(addr) & mask) && --timeout) 55 /* nothing */; 56 57 return !timeout; 58} 59 60#define MODULE_CLKGATE (1 << 30) 61#define MODULE_SFTRST (1 << 31) 62/* 63 * The current mxs_reset_block() will do two things: 64 * [1] enable the module. 65 * [2] reset the module. 66 * 67 * In most of the cases, it's ok. 68 * But in MX23, there is a hardware bug in the BCH block (see erratum #2847). 69 * If you try to soft reset the BCH block, it becomes unusable until 70 * the next hard reset. This case occurs in the NAND boot mode. When the board 71 * boots by NAND, the ROM of the chip will initialize the BCH blocks itself. 72 * So If the driver tries to reset the BCH again, the BCH will not work anymore. 73 * You will see a DMA timeout in this case. The bug has been fixed 74 * in the following chips, such as MX28. 75 * 76 * To avoid this bug, just add a new parameter `just_enable` for 77 * the mxs_reset_block(), and rewrite it here. 78 */ 79static int gpmi_reset_block(void __iomem *reset_addr, bool just_enable) 80{ 81 int ret; 82 int timeout = 0x400; 83 84 /* clear and poll SFTRST */ 85 ret = clear_poll_bit(reset_addr, MODULE_SFTRST); 86 if (unlikely(ret)) 87 goto error; 88 89 /* clear CLKGATE */ 90 writel(MODULE_CLKGATE, reset_addr + MXS_CLR_ADDR); 91 92 if (!just_enable) { 93 /* set SFTRST to reset the block */ 94 writel(MODULE_SFTRST, reset_addr + MXS_SET_ADDR); 95 udelay(1); 96 97 /* poll CLKGATE becoming set */ 98 while ((!(readl(reset_addr) & MODULE_CLKGATE)) && --timeout) 99 /* nothing */; 100 if (unlikely(!timeout)) 101 goto error; 102 } 103 104 /* clear and poll SFTRST */ 105 ret = clear_poll_bit(reset_addr, MODULE_SFTRST); 106 if (unlikely(ret)) 107 goto error; 108 109 /* clear and poll CLKGATE */ 110 ret = clear_poll_bit(reset_addr, MODULE_CLKGATE); 111 if (unlikely(ret)) 112 goto error; 113 114 return 0; 115 116error: 117 pr_err("%s(%p): module reset timeout\n", __func__, reset_addr); 118 return -ETIMEDOUT; 119} 120 121static int __gpmi_enable_clk(struct gpmi_nand_data *this, bool v) 122{ 123 struct clk *clk; 124 int ret; 125 int i; 126 127 for (i = 0; i < GPMI_CLK_MAX; i++) { 128 clk = this->resources.clock[i]; 129 if (!clk) 130 break; 131 132 if (v) { 133 ret = clk_prepare_enable(clk); 134 if (ret) 135 goto err_clk; 136 } else { 137 clk_disable_unprepare(clk); 138 } 139 } 140 return 0; 141 142err_clk: 143 for (; i > 0; i--) 144 clk_disable_unprepare(this->resources.clock[i - 1]); 145 return ret; 146} 147 148#define gpmi_enable_clk(x) __gpmi_enable_clk(x, true) 149#define gpmi_disable_clk(x) __gpmi_enable_clk(x, false) 150 151static int gpmi_init(struct gpmi_nand_data *this) 152{ 153 struct resources *r = &this->resources; 154 int ret; 155 156 ret = pm_runtime_resume_and_get(this->dev); 157 if (ret < 0) 158 return ret; 159 160 ret = gpmi_reset_block(r->gpmi_regs, false); 161 if (ret) 162 goto err_out; 163 164 /* 165 * Reset BCH here, too. We got failures otherwise :( 166 * See later BCH reset for explanation of MX23 and MX28 handling 167 */ 168 ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this)); 169 if (ret) 170 goto err_out; 171 172 /* Choose NAND mode. */ 173 writel(BM_GPMI_CTRL1_GPMI_MODE, r->gpmi_regs + HW_GPMI_CTRL1_CLR); 174 175 /* Set the IRQ polarity. */ 176 writel(BM_GPMI_CTRL1_ATA_IRQRDY_POLARITY, 177 r->gpmi_regs + HW_GPMI_CTRL1_SET); 178 179 /* Disable Write-Protection. */ 180 writel(BM_GPMI_CTRL1_DEV_RESET, r->gpmi_regs + HW_GPMI_CTRL1_SET); 181 182 /* Select BCH ECC. */ 183 writel(BM_GPMI_CTRL1_BCH_MODE, r->gpmi_regs + HW_GPMI_CTRL1_SET); 184 185 /* 186 * Decouple the chip select from dma channel. We use dma0 for all 187 * the chips, force all NAND RDY_BUSY inputs to be sourced from 188 * RDY_BUSY0. 189 */ 190 writel(BM_GPMI_CTRL1_DECOUPLE_CS | BM_GPMI_CTRL1_GANGED_RDYBUSY, 191 r->gpmi_regs + HW_GPMI_CTRL1_SET); 192 193err_out: 194 pm_runtime_put_autosuspend(this->dev); 195 return ret; 196} 197 198/* This function is very useful. It is called only when the bug occur. */ 199static void gpmi_dump_info(struct gpmi_nand_data *this) 200{ 201 struct resources *r = &this->resources; 202 struct bch_geometry *geo = &this->bch_geometry; 203 u32 reg; 204 int i; 205 206 dev_err(this->dev, "Show GPMI registers :\n"); 207 for (i = 0; i <= HW_GPMI_DEBUG / 0x10 + 1; i++) { 208 reg = readl(r->gpmi_regs + i * 0x10); 209 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg); 210 } 211 212 /* start to print out the BCH info */ 213 dev_err(this->dev, "Show BCH registers :\n"); 214 for (i = 0; i <= HW_BCH_VERSION / 0x10 + 1; i++) { 215 reg = readl(r->bch_regs + i * 0x10); 216 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg); 217 } 218 dev_err(this->dev, "BCH Geometry :\n" 219 "GF length : %u\n" 220 "ECC Strength : %u\n" 221 "Page Size in Bytes : %u\n" 222 "Metadata Size in Bytes : %u\n" 223 "ECC0 Chunk Size in Bytes: %u\n" 224 "ECCn Chunk Size in Bytes: %u\n" 225 "ECC Chunk Count : %u\n" 226 "Payload Size in Bytes : %u\n" 227 "Auxiliary Size in Bytes: %u\n" 228 "Auxiliary Status Offset: %u\n" 229 "Block Mark Byte Offset : %u\n" 230 "Block Mark Bit Offset : %u\n", 231 geo->gf_len, 232 geo->ecc_strength, 233 geo->page_size, 234 geo->metadata_size, 235 geo->ecc0_chunk_size, 236 geo->eccn_chunk_size, 237 geo->ecc_chunk_count, 238 geo->payload_size, 239 geo->auxiliary_size, 240 geo->auxiliary_status_offset, 241 geo->block_mark_byte_offset, 242 geo->block_mark_bit_offset); 243} 244 245static bool gpmi_check_ecc(struct gpmi_nand_data *this) 246{ 247 struct nand_chip *chip = &this->nand; 248 struct bch_geometry *geo = &this->bch_geometry; 249 struct nand_device *nand = &chip->base; 250 struct nand_ecc_props *conf = &nand->ecc.ctx.conf; 251 252 conf->step_size = geo->eccn_chunk_size; 253 conf->strength = geo->ecc_strength; 254 255 /* Do the sanity check. */ 256 if (GPMI_IS_MXS(this)) { 257 /* The mx23/mx28 only support the GF13. */ 258 if (geo->gf_len == 14) 259 return false; 260 } 261 262 if (geo->ecc_strength > this->devdata->bch_max_ecc_strength) 263 return false; 264 265 if (!nand_ecc_is_strong_enough(nand)) 266 return false; 267 268 return true; 269} 270 271/* check if bbm locates in data chunk rather than ecc chunk */ 272static bool bbm_in_data_chunk(struct gpmi_nand_data *this, 273 unsigned int *chunk_num) 274{ 275 struct bch_geometry *geo = &this->bch_geometry; 276 struct nand_chip *chip = &this->nand; 277 struct mtd_info *mtd = nand_to_mtd(chip); 278 unsigned int i, j; 279 280 if (geo->ecc0_chunk_size != geo->eccn_chunk_size) { 281 dev_err(this->dev, 282 "The size of ecc0_chunk must equal to eccn_chunk\n"); 283 return false; 284 } 285 286 i = (mtd->writesize * 8 - geo->metadata_size * 8) / 287 (geo->gf_len * geo->ecc_strength + 288 geo->eccn_chunk_size * 8); 289 290 j = (mtd->writesize * 8 - geo->metadata_size * 8) - 291 (geo->gf_len * geo->ecc_strength + 292 geo->eccn_chunk_size * 8) * i; 293 294 if (j < geo->eccn_chunk_size * 8) { 295 *chunk_num = i+1; 296 dev_dbg(this->dev, "Set ecc to %d and bbm in chunk %d\n", 297 geo->ecc_strength, *chunk_num); 298 return true; 299 } 300 301 return false; 302} 303 304/* 305 * If we can get the ECC information from the nand chip, we do not 306 * need to calculate them ourselves. 307 * 308 * We may have available oob space in this case. 309 */ 310static int set_geometry_by_ecc_info(struct gpmi_nand_data *this, 311 unsigned int ecc_strength, 312 unsigned int ecc_step) 313{ 314 struct bch_geometry *geo = &this->bch_geometry; 315 struct nand_chip *chip = &this->nand; 316 struct mtd_info *mtd = nand_to_mtd(chip); 317 unsigned int block_mark_bit_offset; 318 319 switch (ecc_step) { 320 case SZ_512: 321 geo->gf_len = 13; 322 break; 323 case SZ_1K: 324 geo->gf_len = 14; 325 break; 326 default: 327 dev_err(this->dev, 328 "unsupported nand chip. ecc bits : %d, ecc size : %d\n", 329 nanddev_get_ecc_requirements(&chip->base)->strength, 330 nanddev_get_ecc_requirements(&chip->base)->step_size); 331 return -EINVAL; 332 } 333 geo->ecc0_chunk_size = ecc_step; 334 geo->eccn_chunk_size = ecc_step; 335 geo->ecc_strength = round_up(ecc_strength, 2); 336 if (!gpmi_check_ecc(this)) 337 return -EINVAL; 338 339 /* Keep the C >= O */ 340 if (geo->eccn_chunk_size < mtd->oobsize) { 341 dev_err(this->dev, 342 "unsupported nand chip. ecc size: %d, oob size : %d\n", 343 ecc_step, mtd->oobsize); 344 return -EINVAL; 345 } 346 347 /* The default value, see comment in the legacy_set_geometry(). */ 348 geo->metadata_size = 10; 349 350 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size; 351 352 /* 353 * Now, the NAND chip with 2K page(data chunk is 512byte) shows below: 354 * 355 * | P | 356 * |<----------------------------------------------------->| 357 * | | 358 * | (Block Mark) | 359 * | P' | | | | 360 * |<-------------------------------------------->| D | | O' | 361 * | |<---->| |<--->| 362 * V V V V V 363 * +---+----------+-+----------+-+----------+-+----------+-+-----+ 364 * | M | data |E| data |E| data |E| data |E| | 365 * +---+----------+-+----------+-+----------+-+----------+-+-----+ 366 * ^ ^ 367 * | O | 368 * |<------------>| 369 * | | 370 * 371 * P : the page size for BCH module. 372 * E : The ECC strength. 373 * G : the length of Galois Field. 374 * N : The chunk count of per page. 375 * M : the metasize of per page. 376 * C : the ecc chunk size, aka the "data" above. 377 * P': the nand chip's page size. 378 * O : the nand chip's oob size. 379 * O': the free oob. 380 * 381 * The formula for P is : 382 * 383 * E * G * N 384 * P = ------------ + P' + M 385 * 8 386 * 387 * The position of block mark moves forward in the ECC-based view 388 * of page, and the delta is: 389 * 390 * E * G * (N - 1) 391 * D = (---------------- + M) 392 * 8 393 * 394 * Please see the comment in legacy_set_geometry(). 395 * With the condition C >= O , we still can get same result. 396 * So the bit position of the physical block mark within the ECC-based 397 * view of the page is : 398 * (P' - D) * 8 399 */ 400 geo->page_size = mtd->writesize + geo->metadata_size + 401 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8; 402 403 geo->payload_size = mtd->writesize; 404 405 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4); 406 geo->auxiliary_size = ALIGN(geo->metadata_size, 4) 407 + ALIGN(geo->ecc_chunk_count, 4); 408 409 if (!this->swap_block_mark) 410 return 0; 411 412 /* For bit swap. */ 413 block_mark_bit_offset = mtd->writesize * 8 - 414 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1) 415 + geo->metadata_size * 8); 416 417 geo->block_mark_byte_offset = block_mark_bit_offset / 8; 418 geo->block_mark_bit_offset = block_mark_bit_offset % 8; 419 return 0; 420} 421 422/* 423 * Calculate the ECC strength by hand: 424 * E : The ECC strength. 425 * G : the length of Galois Field. 426 * N : The chunk count of per page. 427 * O : the oobsize of the NAND chip. 428 * M : the metasize of per page. 429 * 430 * The formula is : 431 * E * G * N 432 * ------------ <= (O - M) 433 * 8 434 * 435 * So, we get E by: 436 * (O - M) * 8 437 * E <= ------------- 438 * G * N 439 */ 440static inline int get_ecc_strength(struct gpmi_nand_data *this) 441{ 442 struct bch_geometry *geo = &this->bch_geometry; 443 struct mtd_info *mtd = nand_to_mtd(&this->nand); 444 int ecc_strength; 445 446 ecc_strength = ((mtd->oobsize - geo->metadata_size) * 8) 447 / (geo->gf_len * geo->ecc_chunk_count); 448 449 /* We need the minor even number. */ 450 return round_down(ecc_strength, 2); 451} 452 453static int set_geometry_for_large_oob(struct gpmi_nand_data *this) 454{ 455 struct bch_geometry *geo = &this->bch_geometry; 456 struct nand_chip *chip = &this->nand; 457 struct mtd_info *mtd = nand_to_mtd(chip); 458 const struct nand_ecc_props *requirements = 459 nanddev_get_ecc_requirements(&chip->base); 460 unsigned int block_mark_bit_offset; 461 unsigned int max_ecc; 462 unsigned int bbm_chunk; 463 unsigned int i; 464 465 /* sanity check for the minimum ecc nand required */ 466 if (!(requirements->strength > 0 && 467 requirements->step_size > 0)) 468 return -EINVAL; 469 geo->ecc_strength = requirements->strength; 470 471 /* check if platform can support this nand */ 472 if (!gpmi_check_ecc(this)) { 473 dev_err(this->dev, 474 "unsupported NAND chip, minimum ecc required %d\n", 475 geo->ecc_strength); 476 return -EINVAL; 477 } 478 479 /* calculate the maximum ecc platform can support*/ 480 geo->metadata_size = 10; 481 geo->gf_len = 14; 482 geo->ecc0_chunk_size = 1024; 483 geo->eccn_chunk_size = 1024; 484 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size; 485 max_ecc = min(get_ecc_strength(this), 486 this->devdata->bch_max_ecc_strength); 487 488 /* 489 * search a supported ecc strength that makes bbm 490 * located in data chunk 491 */ 492 geo->ecc_strength = max_ecc; 493 while (!(geo->ecc_strength < requirements->strength)) { 494 if (bbm_in_data_chunk(this, &bbm_chunk)) 495 goto geo_setting; 496 geo->ecc_strength -= 2; 497 } 498 499 /* if none of them works, keep using the minimum ecc */ 500 /* nand required but changing ecc page layout */ 501 geo->ecc_strength = requirements->strength; 502 /* add extra ecc for meta data */ 503 geo->ecc0_chunk_size = 0; 504 geo->ecc_chunk_count = (mtd->writesize / geo->eccn_chunk_size) + 1; 505 geo->ecc_for_meta = 1; 506 /* check if oob can afford this extra ecc chunk */ 507 if (mtd->oobsize * 8 < geo->metadata_size * 8 + 508 geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) { 509 dev_err(this->dev, "unsupported NAND chip with new layout\n"); 510 return -EINVAL; 511 } 512 513 /* calculate in which chunk bbm located */ 514 bbm_chunk = (mtd->writesize * 8 - geo->metadata_size * 8 - 515 geo->gf_len * geo->ecc_strength) / 516 (geo->gf_len * geo->ecc_strength + 517 geo->eccn_chunk_size * 8) + 1; 518 519geo_setting: 520 521 geo->page_size = mtd->writesize + geo->metadata_size + 522 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8; 523 geo->payload_size = mtd->writesize; 524 525 /* 526 * The auxiliary buffer contains the metadata and the ECC status. The 527 * metadata is padded to the nearest 32-bit boundary. The ECC status 528 * contains one byte for every ECC chunk, and is also padded to the 529 * nearest 32-bit boundary. 530 */ 531 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4); 532 geo->auxiliary_size = ALIGN(geo->metadata_size, 4) 533 + ALIGN(geo->ecc_chunk_count, 4); 534 535 if (!this->swap_block_mark) 536 return 0; 537 538 /* calculate the number of ecc chunk behind the bbm */ 539 i = (mtd->writesize / geo->eccn_chunk_size) - bbm_chunk + 1; 540 541 block_mark_bit_offset = mtd->writesize * 8 - 542 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - i) 543 + geo->metadata_size * 8); 544 545 geo->block_mark_byte_offset = block_mark_bit_offset / 8; 546 geo->block_mark_bit_offset = block_mark_bit_offset % 8; 547 548 dev_dbg(this->dev, "BCH Geometry :\n" 549 "GF length : %u\n" 550 "ECC Strength : %u\n" 551 "Page Size in Bytes : %u\n" 552 "Metadata Size in Bytes : %u\n" 553 "ECC0 Chunk Size in Bytes: %u\n" 554 "ECCn Chunk Size in Bytes: %u\n" 555 "ECC Chunk Count : %u\n" 556 "Payload Size in Bytes : %u\n" 557 "Auxiliary Size in Bytes: %u\n" 558 "Auxiliary Status Offset: %u\n" 559 "Block Mark Byte Offset : %u\n" 560 "Block Mark Bit Offset : %u\n" 561 "Block Mark in chunk : %u\n" 562 "Ecc for Meta data : %u\n", 563 geo->gf_len, 564 geo->ecc_strength, 565 geo->page_size, 566 geo->metadata_size, 567 geo->ecc0_chunk_size, 568 geo->eccn_chunk_size, 569 geo->ecc_chunk_count, 570 geo->payload_size, 571 geo->auxiliary_size, 572 geo->auxiliary_status_offset, 573 geo->block_mark_byte_offset, 574 geo->block_mark_bit_offset, 575 bbm_chunk, 576 geo->ecc_for_meta); 577 578 return 0; 579} 580 581static int legacy_set_geometry(struct gpmi_nand_data *this) 582{ 583 struct bch_geometry *geo = &this->bch_geometry; 584 struct mtd_info *mtd = nand_to_mtd(&this->nand); 585 unsigned int metadata_size; 586 unsigned int status_size; 587 unsigned int block_mark_bit_offset; 588 589 /* 590 * The size of the metadata can be changed, though we set it to 10 591 * bytes now. But it can't be too large, because we have to save 592 * enough space for BCH. 593 */ 594 geo->metadata_size = 10; 595 596 /* The default for the length of Galois Field. */ 597 geo->gf_len = 13; 598 599 /* The default for chunk size. */ 600 geo->ecc0_chunk_size = 512; 601 geo->eccn_chunk_size = 512; 602 while (geo->eccn_chunk_size < mtd->oobsize) { 603 geo->ecc0_chunk_size *= 2; /* keep C >= O */ 604 geo->eccn_chunk_size *= 2; /* keep C >= O */ 605 geo->gf_len = 14; 606 } 607 608 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size; 609 610 /* We use the same ECC strength for all chunks. */ 611 geo->ecc_strength = get_ecc_strength(this); 612 if (!gpmi_check_ecc(this)) { 613 dev_err(this->dev, 614 "ecc strength: %d cannot be supported by the controller (%d)\n" 615 "try to use minimum ecc strength that NAND chip required\n", 616 geo->ecc_strength, 617 this->devdata->bch_max_ecc_strength); 618 return -EINVAL; 619 } 620 621 geo->page_size = mtd->writesize + geo->metadata_size + 622 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8; 623 geo->payload_size = mtd->writesize; 624 625 /* 626 * The auxiliary buffer contains the metadata and the ECC status. The 627 * metadata is padded to the nearest 32-bit boundary. The ECC status 628 * contains one byte for every ECC chunk, and is also padded to the 629 * nearest 32-bit boundary. 630 */ 631 metadata_size = ALIGN(geo->metadata_size, 4); 632 status_size = ALIGN(geo->ecc_chunk_count, 4); 633 634 geo->auxiliary_size = metadata_size + status_size; 635 geo->auxiliary_status_offset = metadata_size; 636 637 if (!this->swap_block_mark) 638 return 0; 639 640 /* 641 * We need to compute the byte and bit offsets of 642 * the physical block mark within the ECC-based view of the page. 643 * 644 * NAND chip with 2K page shows below: 645 * (Block Mark) 646 * | | 647 * | D | 648 * |<---->| 649 * V V 650 * +---+----------+-+----------+-+----------+-+----------+-+ 651 * | M | data |E| data |E| data |E| data |E| 652 * +---+----------+-+----------+-+----------+-+----------+-+ 653 * 654 * The position of block mark moves forward in the ECC-based view 655 * of page, and the delta is: 656 * 657 * E * G * (N - 1) 658 * D = (---------------- + M) 659 * 8 660 * 661 * With the formula to compute the ECC strength, and the condition 662 * : C >= O (C is the ecc chunk size) 663 * 664 * It's easy to deduce to the following result: 665 * 666 * E * G (O - M) C - M C - M 667 * ----------- <= ------- <= -------- < --------- 668 * 8 N N (N - 1) 669 * 670 * So, we get: 671 * 672 * E * G * (N - 1) 673 * D = (---------------- + M) < C 674 * 8 675 * 676 * The above inequality means the position of block mark 677 * within the ECC-based view of the page is still in the data chunk, 678 * and it's NOT in the ECC bits of the chunk. 679 * 680 * Use the following to compute the bit position of the 681 * physical block mark within the ECC-based view of the page: 682 * (page_size - D) * 8 683 * 684 * --Huang Shijie 685 */ 686 block_mark_bit_offset = mtd->writesize * 8 - 687 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1) 688 + geo->metadata_size * 8); 689 690 geo->block_mark_byte_offset = block_mark_bit_offset / 8; 691 geo->block_mark_bit_offset = block_mark_bit_offset % 8; 692 return 0; 693} 694 695static int common_nfc_set_geometry(struct gpmi_nand_data *this) 696{ 697 struct nand_chip *chip = &this->nand; 698 struct mtd_info *mtd = nand_to_mtd(&this->nand); 699 const struct nand_ecc_props *requirements = 700 nanddev_get_ecc_requirements(&chip->base); 701 bool use_minimun_ecc; 702 int err; 703 704 use_minimun_ecc = of_property_read_bool(this->dev->of_node, 705 "fsl,use-minimum-ecc"); 706 707 /* use legacy bch geometry settings by default*/ 708 if ((!use_minimun_ecc && mtd->oobsize < 1024) || 709 !(requirements->strength > 0 && requirements->step_size > 0)) { 710 dev_dbg(this->dev, "use legacy bch geometry\n"); 711 err = legacy_set_geometry(this); 712 if (!err) 713 return 0; 714 } 715 716 /* for large oob nand */ 717 if (mtd->oobsize > 1024) { 718 dev_dbg(this->dev, "use large oob bch geometry\n"); 719 err = set_geometry_for_large_oob(this); 720 if (!err) 721 return 0; 722 } 723 724 /* otherwise use the minimum ecc nand chip required */ 725 dev_dbg(this->dev, "use minimum ecc bch geometry\n"); 726 err = set_geometry_by_ecc_info(this, requirements->strength, 727 requirements->step_size); 728 if (err) 729 dev_err(this->dev, "none of the bch geometry setting works\n"); 730 731 return err; 732} 733 734/* Configures the geometry for BCH. */ 735static int bch_set_geometry(struct gpmi_nand_data *this) 736{ 737 struct resources *r = &this->resources; 738 int ret; 739 740 ret = common_nfc_set_geometry(this); 741 if (ret) 742 return ret; 743 744 ret = pm_runtime_resume_and_get(this->dev); 745 if (ret < 0) { 746 return ret; 747 } 748 749 /* 750 * Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this 751 * chip, otherwise it will lock up. So we skip resetting BCH on the MX23. 752 * and MX28. 753 */ 754 ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this)); 755 if (ret) 756 goto err_out; 757 758 /* Set *all* chip selects to use layout 0. */ 759 writel(0, r->bch_regs + HW_BCH_LAYOUTSELECT); 760 761 ret = 0; 762err_out: 763 pm_runtime_put_autosuspend(this->dev); 764 765 return ret; 766} 767 768/* 769 * <1> Firstly, we should know what's the GPMI-clock means. 770 * The GPMI-clock is the internal clock in the gpmi nand controller. 771 * If you set 100MHz to gpmi nand controller, the GPMI-clock's period 772 * is 10ns. Mark the GPMI-clock's period as GPMI-clock-period. 773 * 774 * <2> Secondly, we should know what's the frequency on the nand chip pins. 775 * The frequency on the nand chip pins is derived from the GPMI-clock. 776 * We can get it from the following equation: 777 * 778 * F = G / (DS + DH) 779 * 780 * F : the frequency on the nand chip pins. 781 * G : the GPMI clock, such as 100MHz. 782 * DS : GPMI_HW_GPMI_TIMING0:DATA_SETUP 783 * DH : GPMI_HW_GPMI_TIMING0:DATA_HOLD 784 * 785 * <3> Thirdly, when the frequency on the nand chip pins is above 33MHz, 786 * the nand EDO(extended Data Out) timing could be applied. 787 * The GPMI implements a feedback read strobe to sample the read data. 788 * The feedback read strobe can be delayed to support the nand EDO timing 789 * where the read strobe may deasserts before the read data is valid, and 790 * read data is valid for some time after read strobe. 791 * 792 * The following figure illustrates some aspects of a NAND Flash read: 793 * 794 * |<---tREA---->| 795 * | | 796 * | | | 797 * |<--tRP-->| | 798 * | | | 799 * __ ___|__________________________________ 800 * RDN \________/ | 801 * | 802 * /---------\ 803 * Read Data --------------< >--------- 804 * \---------/ 805 * | | 806 * |<-D->| 807 * FeedbackRDN ________ ____________ 808 * \___________/ 809 * 810 * D stands for delay, set in the HW_GPMI_CTRL1:RDN_DELAY. 811 * 812 * 813 * <4> Now, we begin to describe how to compute the right RDN_DELAY. 814 * 815 * 4.1) From the aspect of the nand chip pins: 816 * Delay = (tREA + C - tRP) {1} 817 * 818 * tREA : the maximum read access time. 819 * C : a constant to adjust the delay. default is 4000ps. 820 * tRP : the read pulse width, which is exactly: 821 * tRP = (GPMI-clock-period) * DATA_SETUP 822 * 823 * 4.2) From the aspect of the GPMI nand controller: 824 * Delay = RDN_DELAY * 0.125 * RP {2} 825 * 826 * RP : the DLL reference period. 827 * if (GPMI-clock-period > DLL_THRETHOLD) 828 * RP = GPMI-clock-period / 2; 829 * else 830 * RP = GPMI-clock-period; 831 * 832 * Set the HW_GPMI_CTRL1:HALF_PERIOD if GPMI-clock-period 833 * is greater DLL_THRETHOLD. In other SOCs, the DLL_THRETHOLD 834 * is 16000ps, but in mx6q, we use 12000ps. 835 * 836 * 4.3) since {1} equals {2}, we get: 837 * 838 * (tREA + 4000 - tRP) * 8 839 * RDN_DELAY = ----------------------- {3} 840 * RP 841 */ 842static int gpmi_nfc_compute_timings(struct gpmi_nand_data *this, 843 const struct nand_sdr_timings *sdr) 844{ 845 struct gpmi_nfc_hardware_timing *hw = &this->hw; 846 struct resources *r = &this->resources; 847 unsigned int dll_threshold_ps = this->devdata->max_chain_delay; 848 unsigned int period_ps, reference_period_ps; 849 unsigned int data_setup_cycles, data_hold_cycles, addr_setup_cycles; 850 unsigned int tRP_ps; 851 bool use_half_period; 852 int sample_delay_ps, sample_delay_factor; 853 unsigned int busy_timeout_cycles; 854 u8 wrn_dly_sel; 855 unsigned long clk_rate, min_rate; 856 u64 busy_timeout_ps; 857 858 if (sdr->tRC_min >= 30000) { 859 /* ONFI non-EDO modes [0-3] */ 860 hw->clk_rate = 22000000; 861 min_rate = 0; 862 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_4_TO_8NS; 863 } else if (sdr->tRC_min >= 25000) { 864 /* ONFI EDO mode 4 */ 865 hw->clk_rate = 80000000; 866 min_rate = 22000000; 867 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY; 868 } else { 869 /* ONFI EDO mode 5 */ 870 hw->clk_rate = 100000000; 871 min_rate = 80000000; 872 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY; 873 } 874 875 clk_rate = clk_round_rate(r->clock[0], hw->clk_rate); 876 if (clk_rate <= min_rate) { 877 dev_err(this->dev, "clock setting: expected %ld, got %ld\n", 878 hw->clk_rate, clk_rate); 879 return -ENOTSUPP; 880 } 881 882 hw->clk_rate = clk_rate; 883 /* SDR core timings are given in picoseconds */ 884 period_ps = div_u64((u64)NSEC_PER_SEC * 1000, hw->clk_rate); 885 886 addr_setup_cycles = TO_CYCLES(sdr->tALS_min, period_ps); 887 data_setup_cycles = TO_CYCLES(sdr->tDS_min, period_ps); 888 data_hold_cycles = TO_CYCLES(sdr->tDH_min, period_ps); 889 busy_timeout_ps = max(sdr->tBERS_max, sdr->tPROG_max); 890 busy_timeout_cycles = TO_CYCLES(busy_timeout_ps, period_ps); 891 892 hw->timing0 = BF_GPMI_TIMING0_ADDRESS_SETUP(addr_setup_cycles) | 893 BF_GPMI_TIMING0_DATA_HOLD(data_hold_cycles) | 894 BF_GPMI_TIMING0_DATA_SETUP(data_setup_cycles); 895 hw->timing1 = BF_GPMI_TIMING1_BUSY_TIMEOUT(DIV_ROUND_UP(busy_timeout_cycles, 4096)); 896 897 /* 898 * Derive NFC ideal delay from {3}: 899 * 900 * (tREA + 4000 - tRP) * 8 901 * RDN_DELAY = ----------------------- 902 * RP 903 */ 904 if (period_ps > dll_threshold_ps) { 905 use_half_period = true; 906 reference_period_ps = period_ps / 2; 907 } else { 908 use_half_period = false; 909 reference_period_ps = period_ps; 910 } 911 912 tRP_ps = data_setup_cycles * period_ps; 913 sample_delay_ps = (sdr->tREA_max + 4000 - tRP_ps) * 8; 914 if (sample_delay_ps > 0) 915 sample_delay_factor = sample_delay_ps / reference_period_ps; 916 else 917 sample_delay_factor = 0; 918 919 hw->ctrl1n = BF_GPMI_CTRL1_WRN_DLY_SEL(wrn_dly_sel); 920 if (sample_delay_factor) 921 hw->ctrl1n |= BF_GPMI_CTRL1_RDN_DELAY(sample_delay_factor) | 922 BM_GPMI_CTRL1_DLL_ENABLE | 923 (use_half_period ? BM_GPMI_CTRL1_HALF_PERIOD : 0); 924 return 0; 925} 926 927static int gpmi_nfc_apply_timings(struct gpmi_nand_data *this) 928{ 929 struct gpmi_nfc_hardware_timing *hw = &this->hw; 930 struct resources *r = &this->resources; 931 void __iomem *gpmi_regs = r->gpmi_regs; 932 unsigned int dll_wait_time_us; 933 int ret; 934 935 /* Clock dividers do NOT guarantee a clean clock signal on its output 936 * during the change of the divide factor on i.MX6Q/UL/SX. On i.MX7/8, 937 * all clock dividers provide these guarantee. 938 */ 939 if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this)) 940 clk_disable_unprepare(r->clock[0]); 941 942 ret = clk_set_rate(r->clock[0], hw->clk_rate); 943 if (ret) { 944 dev_err(this->dev, "cannot set clock rate to %lu Hz: %d\n", hw->clk_rate, ret); 945 return ret; 946 } 947 948 if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this)) { 949 ret = clk_prepare_enable(r->clock[0]); 950 if (ret) 951 return ret; 952 } 953 954 writel(hw->timing0, gpmi_regs + HW_GPMI_TIMING0); 955 writel(hw->timing1, gpmi_regs + HW_GPMI_TIMING1); 956 957 /* 958 * Clear several CTRL1 fields, DLL must be disabled when setting 959 * RDN_DELAY or HALF_PERIOD. 960 */ 961 writel(BM_GPMI_CTRL1_CLEAR_MASK, gpmi_regs + HW_GPMI_CTRL1_CLR); 962 writel(hw->ctrl1n, gpmi_regs + HW_GPMI_CTRL1_SET); 963 964 /* Wait 64 clock cycles before using the GPMI after enabling the DLL */ 965 dll_wait_time_us = USEC_PER_SEC / hw->clk_rate * 64; 966 if (!dll_wait_time_us) 967 dll_wait_time_us = 1; 968 969 /* Wait for the DLL to settle. */ 970 udelay(dll_wait_time_us); 971 972 return 0; 973} 974 975static int gpmi_setup_interface(struct nand_chip *chip, int chipnr, 976 const struct nand_interface_config *conf) 977{ 978 struct gpmi_nand_data *this = nand_get_controller_data(chip); 979 const struct nand_sdr_timings *sdr; 980 int ret; 981 982 /* Retrieve required NAND timings */ 983 sdr = nand_get_sdr_timings(conf); 984 if (IS_ERR(sdr)) 985 return PTR_ERR(sdr); 986 987 /* Only MX28/MX6 GPMI controller can reach EDO timings */ 988 if (sdr->tRC_min <= 25000 && !this->devdata->support_edo_timing) 989 return -ENOTSUPP; 990 991 /* Stop here if this call was just a check */ 992 if (chipnr < 0) 993 return 0; 994 995 /* Do the actual derivation of the controller timings */ 996 ret = gpmi_nfc_compute_timings(this, sdr); 997 if (ret) 998 return ret; 999 1000 this->hw.must_apply_timings = true; 1001 1002 return 0; 1003} 1004 1005/* Clears a BCH interrupt. */ 1006static void gpmi_clear_bch(struct gpmi_nand_data *this) 1007{ 1008 struct resources *r = &this->resources; 1009 writel(BM_BCH_CTRL_COMPLETE_IRQ, r->bch_regs + HW_BCH_CTRL_CLR); 1010} 1011 1012static struct dma_chan *get_dma_chan(struct gpmi_nand_data *this) 1013{ 1014 /* We use the DMA channel 0 to access all the nand chips. */ 1015 return this->dma_chans[0]; 1016} 1017 1018/* This will be called after the DMA operation is finished. */ 1019static void dma_irq_callback(void *param) 1020{ 1021 struct gpmi_nand_data *this = param; 1022 struct completion *dma_c = &this->dma_done; 1023 1024 complete(dma_c); 1025} 1026 1027static irqreturn_t bch_irq(int irq, void *cookie) 1028{ 1029 struct gpmi_nand_data *this = cookie; 1030 1031 gpmi_clear_bch(this); 1032 complete(&this->bch_done); 1033 return IRQ_HANDLED; 1034} 1035 1036static int gpmi_raw_len_to_len(struct gpmi_nand_data *this, int raw_len) 1037{ 1038 /* 1039 * raw_len is the length to read/write including bch data which 1040 * we are passed in exec_op. Calculate the data length from it. 1041 */ 1042 if (this->bch) 1043 return ALIGN_DOWN(raw_len, this->bch_geometry.eccn_chunk_size); 1044 else 1045 return raw_len; 1046} 1047 1048/* Can we use the upper's buffer directly for DMA? */ 1049static bool prepare_data_dma(struct gpmi_nand_data *this, const void *buf, 1050 int raw_len, struct scatterlist *sgl, 1051 enum dma_data_direction dr) 1052{ 1053 int ret; 1054 int len = gpmi_raw_len_to_len(this, raw_len); 1055 1056 /* first try to map the upper buffer directly */ 1057 if (virt_addr_valid(buf) && !object_is_on_stack(buf)) { 1058 sg_init_one(sgl, buf, len); 1059 ret = dma_map_sg(this->dev, sgl, 1, dr); 1060 if (ret == 0) 1061 goto map_fail; 1062 1063 return true; 1064 } 1065 1066map_fail: 1067 /* We have to use our own DMA buffer. */ 1068 sg_init_one(sgl, this->data_buffer_dma, len); 1069 1070 if (dr == DMA_TO_DEVICE && buf != this->data_buffer_dma) 1071 memcpy(this->data_buffer_dma, buf, len); 1072 1073 dma_map_sg(this->dev, sgl, 1, dr); 1074 1075 return false; 1076} 1077 1078/* add our owner bbt descriptor */ 1079static uint8_t scan_ff_pattern[] = { 0xff }; 1080static struct nand_bbt_descr gpmi_bbt_descr = { 1081 .options = 0, 1082 .offs = 0, 1083 .len = 1, 1084 .pattern = scan_ff_pattern 1085}; 1086 1087/* 1088 * We may change the layout if we can get the ECC info from the datasheet, 1089 * else we will use all the (page + OOB). 1090 */ 1091static int gpmi_ooblayout_ecc(struct mtd_info *mtd, int section, 1092 struct mtd_oob_region *oobregion) 1093{ 1094 struct nand_chip *chip = mtd_to_nand(mtd); 1095 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1096 struct bch_geometry *geo = &this->bch_geometry; 1097 1098 if (section) 1099 return -ERANGE; 1100 1101 oobregion->offset = 0; 1102 oobregion->length = geo->page_size - mtd->writesize; 1103 1104 return 0; 1105} 1106 1107static int gpmi_ooblayout_free(struct mtd_info *mtd, int section, 1108 struct mtd_oob_region *oobregion) 1109{ 1110 struct nand_chip *chip = mtd_to_nand(mtd); 1111 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1112 struct bch_geometry *geo = &this->bch_geometry; 1113 1114 if (section) 1115 return -ERANGE; 1116 1117 /* The available oob size we have. */ 1118 if (geo->page_size < mtd->writesize + mtd->oobsize) { 1119 oobregion->offset = geo->page_size - mtd->writesize; 1120 oobregion->length = mtd->oobsize - oobregion->offset; 1121 } 1122 1123 return 0; 1124} 1125 1126static const char * const gpmi_clks_for_mx2x[] = { 1127 "gpmi_io", 1128}; 1129 1130static const struct mtd_ooblayout_ops gpmi_ooblayout_ops = { 1131 .ecc = gpmi_ooblayout_ecc, 1132 .free = gpmi_ooblayout_free, 1133}; 1134 1135static const struct gpmi_devdata gpmi_devdata_imx23 = { 1136 .type = IS_MX23, 1137 .bch_max_ecc_strength = 20, 1138 .max_chain_delay = 16000, 1139 .clks = gpmi_clks_for_mx2x, 1140 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x), 1141}; 1142 1143static const struct gpmi_devdata gpmi_devdata_imx28 = { 1144 .type = IS_MX28, 1145 .bch_max_ecc_strength = 20, 1146 .max_chain_delay = 16000, 1147 .support_edo_timing = true, 1148 .clks = gpmi_clks_for_mx2x, 1149 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x), 1150}; 1151 1152static const char * const gpmi_clks_for_mx6[] = { 1153 "gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", "per1_bch", 1154}; 1155 1156static const struct gpmi_devdata gpmi_devdata_imx6q = { 1157 .type = IS_MX6Q, 1158 .bch_max_ecc_strength = 40, 1159 .max_chain_delay = 12000, 1160 .support_edo_timing = true, 1161 .clks = gpmi_clks_for_mx6, 1162 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6), 1163}; 1164 1165static const struct gpmi_devdata gpmi_devdata_imx6sx = { 1166 .type = IS_MX6SX, 1167 .bch_max_ecc_strength = 62, 1168 .max_chain_delay = 12000, 1169 .support_edo_timing = true, 1170 .clks = gpmi_clks_for_mx6, 1171 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6), 1172}; 1173 1174static const char * const gpmi_clks_for_mx7d[] = { 1175 "gpmi_io", "gpmi_bch_apb", 1176}; 1177 1178static const struct gpmi_devdata gpmi_devdata_imx7d = { 1179 .type = IS_MX7D, 1180 .bch_max_ecc_strength = 62, 1181 .max_chain_delay = 12000, 1182 .support_edo_timing = true, 1183 .clks = gpmi_clks_for_mx7d, 1184 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx7d), 1185}; 1186 1187static const char *gpmi_clks_for_mx8qxp[GPMI_CLK_MAX] = { 1188 "gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", 1189}; 1190 1191static const struct gpmi_devdata gpmi_devdata_imx8qxp = { 1192 .type = IS_MX8QXP, 1193 .bch_max_ecc_strength = 62, 1194 .max_chain_delay = 12000, 1195 .support_edo_timing = true, 1196 .clks = gpmi_clks_for_mx8qxp, 1197 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx8qxp), 1198}; 1199 1200static int acquire_register_block(struct gpmi_nand_data *this, 1201 const char *res_name) 1202{ 1203 struct platform_device *pdev = this->pdev; 1204 struct resources *res = &this->resources; 1205 void __iomem *p; 1206 1207 p = devm_platform_ioremap_resource_byname(pdev, res_name); 1208 if (IS_ERR(p)) 1209 return PTR_ERR(p); 1210 1211 if (!strcmp(res_name, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME)) 1212 res->gpmi_regs = p; 1213 else if (!strcmp(res_name, GPMI_NAND_BCH_REGS_ADDR_RES_NAME)) 1214 res->bch_regs = p; 1215 else 1216 dev_err(this->dev, "unknown resource name : %s\n", res_name); 1217 1218 return 0; 1219} 1220 1221static int acquire_bch_irq(struct gpmi_nand_data *this, irq_handler_t irq_h) 1222{ 1223 struct platform_device *pdev = this->pdev; 1224 const char *res_name = GPMI_NAND_BCH_INTERRUPT_RES_NAME; 1225 int err; 1226 1227 err = platform_get_irq_byname(pdev, res_name); 1228 if (err < 0) 1229 return err; 1230 1231 err = devm_request_irq(this->dev, err, irq_h, 0, res_name, this); 1232 if (err) 1233 dev_err(this->dev, "error requesting BCH IRQ\n"); 1234 1235 return err; 1236} 1237 1238static void release_dma_channels(struct gpmi_nand_data *this) 1239{ 1240 unsigned int i; 1241 for (i = 0; i < DMA_CHANS; i++) 1242 if (this->dma_chans[i]) { 1243 dma_release_channel(this->dma_chans[i]); 1244 this->dma_chans[i] = NULL; 1245 } 1246} 1247 1248static int acquire_dma_channels(struct gpmi_nand_data *this) 1249{ 1250 struct platform_device *pdev = this->pdev; 1251 struct dma_chan *dma_chan; 1252 int ret = 0; 1253 1254 /* request dma channel */ 1255 dma_chan = dma_request_chan(&pdev->dev, "rx-tx"); 1256 if (IS_ERR(dma_chan)) { 1257 ret = dev_err_probe(this->dev, PTR_ERR(dma_chan), 1258 "DMA channel request failed\n"); 1259 release_dma_channels(this); 1260 } else { 1261 this->dma_chans[0] = dma_chan; 1262 } 1263 1264 return ret; 1265} 1266 1267static int gpmi_get_clks(struct gpmi_nand_data *this) 1268{ 1269 struct resources *r = &this->resources; 1270 struct clk *clk; 1271 int err, i; 1272 1273 for (i = 0; i < this->devdata->clks_count; i++) { 1274 clk = devm_clk_get(this->dev, this->devdata->clks[i]); 1275 if (IS_ERR(clk)) { 1276 err = PTR_ERR(clk); 1277 goto err_clock; 1278 } 1279 1280 r->clock[i] = clk; 1281 } 1282 1283 return 0; 1284 1285err_clock: 1286 dev_dbg(this->dev, "failed in finding the clocks.\n"); 1287 return err; 1288} 1289 1290static int acquire_resources(struct gpmi_nand_data *this) 1291{ 1292 int ret; 1293 1294 ret = acquire_register_block(this, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME); 1295 if (ret) 1296 goto exit_regs; 1297 1298 ret = acquire_register_block(this, GPMI_NAND_BCH_REGS_ADDR_RES_NAME); 1299 if (ret) 1300 goto exit_regs; 1301 1302 ret = acquire_bch_irq(this, bch_irq); 1303 if (ret) 1304 goto exit_regs; 1305 1306 ret = acquire_dma_channels(this); 1307 if (ret) 1308 goto exit_regs; 1309 1310 ret = gpmi_get_clks(this); 1311 if (ret) 1312 goto exit_clock; 1313 return 0; 1314 1315exit_clock: 1316 release_dma_channels(this); 1317exit_regs: 1318 return ret; 1319} 1320 1321static void release_resources(struct gpmi_nand_data *this) 1322{ 1323 release_dma_channels(this); 1324} 1325 1326static void gpmi_free_dma_buffer(struct gpmi_nand_data *this) 1327{ 1328 struct device *dev = this->dev; 1329 struct bch_geometry *geo = &this->bch_geometry; 1330 1331 if (this->auxiliary_virt && virt_addr_valid(this->auxiliary_virt)) 1332 dma_free_coherent(dev, geo->auxiliary_size, 1333 this->auxiliary_virt, 1334 this->auxiliary_phys); 1335 kfree(this->data_buffer_dma); 1336 kfree(this->raw_buffer); 1337 1338 this->data_buffer_dma = NULL; 1339 this->raw_buffer = NULL; 1340} 1341 1342/* Allocate the DMA buffers */ 1343static int gpmi_alloc_dma_buffer(struct gpmi_nand_data *this) 1344{ 1345 struct bch_geometry *geo = &this->bch_geometry; 1346 struct device *dev = this->dev; 1347 struct mtd_info *mtd = nand_to_mtd(&this->nand); 1348 1349 /* 1350 * [2] Allocate a read/write data buffer. 1351 * The gpmi_alloc_dma_buffer can be called twice. 1352 * We allocate a PAGE_SIZE length buffer if gpmi_alloc_dma_buffer 1353 * is called before the NAND identification; and we allocate a 1354 * buffer of the real NAND page size when the gpmi_alloc_dma_buffer 1355 * is called after. 1356 */ 1357 this->data_buffer_dma = kzalloc(mtd->writesize ?: PAGE_SIZE, 1358 GFP_DMA | GFP_KERNEL); 1359 if (this->data_buffer_dma == NULL) 1360 goto error_alloc; 1361 1362 this->auxiliary_virt = dma_alloc_coherent(dev, geo->auxiliary_size, 1363 &this->auxiliary_phys, GFP_DMA); 1364 if (!this->auxiliary_virt) 1365 goto error_alloc; 1366 1367 this->raw_buffer = kzalloc((mtd->writesize ?: PAGE_SIZE) + mtd->oobsize, GFP_KERNEL); 1368 if (!this->raw_buffer) 1369 goto error_alloc; 1370 1371 return 0; 1372 1373error_alloc: 1374 gpmi_free_dma_buffer(this); 1375 return -ENOMEM; 1376} 1377 1378/* 1379 * Handles block mark swapping. 1380 * It can be called in swapping the block mark, or swapping it back, 1381 * because the operations are the same. 1382 */ 1383static void block_mark_swapping(struct gpmi_nand_data *this, 1384 void *payload, void *auxiliary) 1385{ 1386 struct bch_geometry *nfc_geo = &this->bch_geometry; 1387 unsigned char *p; 1388 unsigned char *a; 1389 unsigned int bit; 1390 unsigned char mask; 1391 unsigned char from_data; 1392 unsigned char from_oob; 1393 1394 if (!this->swap_block_mark) 1395 return; 1396 1397 /* 1398 * If control arrives here, we're swapping. Make some convenience 1399 * variables. 1400 */ 1401 bit = nfc_geo->block_mark_bit_offset; 1402 p = payload + nfc_geo->block_mark_byte_offset; 1403 a = auxiliary; 1404 1405 /* 1406 * Get the byte from the data area that overlays the block mark. Since 1407 * the ECC engine applies its own view to the bits in the page, the 1408 * physical block mark won't (in general) appear on a byte boundary in 1409 * the data. 1410 */ 1411 from_data = (p[0] >> bit) | (p[1] << (8 - bit)); 1412 1413 /* Get the byte from the OOB. */ 1414 from_oob = a[0]; 1415 1416 /* Swap them. */ 1417 a[0] = from_data; 1418 1419 mask = (0x1 << bit) - 1; 1420 p[0] = (p[0] & mask) | (from_oob << bit); 1421 1422 mask = ~0 << bit; 1423 p[1] = (p[1] & mask) | (from_oob >> (8 - bit)); 1424} 1425 1426static int gpmi_count_bitflips(struct nand_chip *chip, void *buf, int first, 1427 int last, int meta) 1428{ 1429 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1430 struct bch_geometry *nfc_geo = &this->bch_geometry; 1431 struct mtd_info *mtd = nand_to_mtd(chip); 1432 int i; 1433 unsigned char *status; 1434 unsigned int max_bitflips = 0; 1435 1436 /* Loop over status bytes, accumulating ECC status. */ 1437 status = this->auxiliary_virt + ALIGN(meta, 4); 1438 1439 for (i = first; i < last; i++, status++) { 1440 if ((*status == STATUS_GOOD) || (*status == STATUS_ERASED)) 1441 continue; 1442 1443 if (*status == STATUS_UNCORRECTABLE) { 1444 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len; 1445 u8 *eccbuf = this->raw_buffer; 1446 int offset, bitoffset; 1447 int eccbytes; 1448 int flips; 1449 1450 /* Read ECC bytes into our internal raw_buffer */ 1451 offset = nfc_geo->metadata_size * 8; 1452 offset += ((8 * nfc_geo->eccn_chunk_size) + eccbits) * (i + 1); 1453 offset -= eccbits; 1454 bitoffset = offset % 8; 1455 eccbytes = DIV_ROUND_UP(offset + eccbits, 8); 1456 offset /= 8; 1457 eccbytes -= offset; 1458 nand_change_read_column_op(chip, offset, eccbuf, 1459 eccbytes, false); 1460 1461 /* 1462 * ECC data are not byte aligned and we may have 1463 * in-band data in the first and last byte of 1464 * eccbuf. Set non-eccbits to one so that 1465 * nand_check_erased_ecc_chunk() does not count them 1466 * as bitflips. 1467 */ 1468 if (bitoffset) 1469 eccbuf[0] |= GENMASK(bitoffset - 1, 0); 1470 1471 bitoffset = (bitoffset + eccbits) % 8; 1472 if (bitoffset) 1473 eccbuf[eccbytes - 1] |= GENMASK(7, bitoffset); 1474 1475 /* 1476 * The ECC hardware has an uncorrectable ECC status 1477 * code in case we have bitflips in an erased page. As 1478 * nothing was written into this subpage the ECC is 1479 * obviously wrong and we can not trust it. We assume 1480 * at this point that we are reading an erased page and 1481 * try to correct the bitflips in buffer up to 1482 * ecc_strength bitflips. If this is a page with random 1483 * data, we exceed this number of bitflips and have a 1484 * ECC failure. Otherwise we use the corrected buffer. 1485 */ 1486 if (i == 0) { 1487 /* The first block includes metadata */ 1488 flips = nand_check_erased_ecc_chunk( 1489 buf + i * nfc_geo->eccn_chunk_size, 1490 nfc_geo->eccn_chunk_size, 1491 eccbuf, eccbytes, 1492 this->auxiliary_virt, 1493 nfc_geo->metadata_size, 1494 nfc_geo->ecc_strength); 1495 } else { 1496 flips = nand_check_erased_ecc_chunk( 1497 buf + i * nfc_geo->eccn_chunk_size, 1498 nfc_geo->eccn_chunk_size, 1499 eccbuf, eccbytes, 1500 NULL, 0, 1501 nfc_geo->ecc_strength); 1502 } 1503 1504 if (flips > 0) { 1505 max_bitflips = max_t(unsigned int, max_bitflips, 1506 flips); 1507 mtd->ecc_stats.corrected += flips; 1508 continue; 1509 } 1510 1511 mtd->ecc_stats.failed++; 1512 continue; 1513 } 1514 1515 mtd->ecc_stats.corrected += *status; 1516 max_bitflips = max_t(unsigned int, max_bitflips, *status); 1517 } 1518 1519 return max_bitflips; 1520} 1521 1522static void gpmi_bch_layout_std(struct gpmi_nand_data *this) 1523{ 1524 struct bch_geometry *geo = &this->bch_geometry; 1525 unsigned int ecc_strength = geo->ecc_strength >> 1; 1526 unsigned int gf_len = geo->gf_len; 1527 unsigned int block0_size = geo->ecc0_chunk_size; 1528 unsigned int blockn_size = geo->eccn_chunk_size; 1529 1530 this->bch_flashlayout0 = 1531 BF_BCH_FLASH0LAYOUT0_NBLOCKS(geo->ecc_chunk_count - 1) | 1532 BF_BCH_FLASH0LAYOUT0_META_SIZE(geo->metadata_size) | 1533 BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) | 1534 BF_BCH_FLASH0LAYOUT0_GF(gf_len, this) | 1535 BF_BCH_FLASH0LAYOUT0_DATA0_SIZE(block0_size, this); 1536 1537 this->bch_flashlayout1 = 1538 BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(geo->page_size) | 1539 BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) | 1540 BF_BCH_FLASH0LAYOUT1_GF(gf_len, this) | 1541 BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(blockn_size, this); 1542} 1543 1544static int gpmi_ecc_read_page(struct nand_chip *chip, uint8_t *buf, 1545 int oob_required, int page) 1546{ 1547 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1548 struct mtd_info *mtd = nand_to_mtd(chip); 1549 struct bch_geometry *geo = &this->bch_geometry; 1550 unsigned int max_bitflips; 1551 int ret; 1552 1553 gpmi_bch_layout_std(this); 1554 this->bch = true; 1555 1556 ret = nand_read_page_op(chip, page, 0, buf, geo->page_size); 1557 if (ret) 1558 return ret; 1559 1560 max_bitflips = gpmi_count_bitflips(chip, buf, 0, 1561 geo->ecc_chunk_count, 1562 geo->auxiliary_status_offset); 1563 1564 /* handle the block mark swapping */ 1565 block_mark_swapping(this, buf, this->auxiliary_virt); 1566 1567 if (oob_required) { 1568 /* 1569 * It's time to deliver the OOB bytes. See gpmi_ecc_read_oob() 1570 * for details about our policy for delivering the OOB. 1571 * 1572 * We fill the caller's buffer with set bits, and then copy the 1573 * block mark to th caller's buffer. Note that, if block mark 1574 * swapping was necessary, it has already been done, so we can 1575 * rely on the first byte of the auxiliary buffer to contain 1576 * the block mark. 1577 */ 1578 memset(chip->oob_poi, ~0, mtd->oobsize); 1579 chip->oob_poi[0] = ((uint8_t *)this->auxiliary_virt)[0]; 1580 } 1581 1582 return max_bitflips; 1583} 1584 1585/* Fake a virtual small page for the subpage read */ 1586static int gpmi_ecc_read_subpage(struct nand_chip *chip, uint32_t offs, 1587 uint32_t len, uint8_t *buf, int page) 1588{ 1589 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1590 struct bch_geometry *geo = &this->bch_geometry; 1591 int size = chip->ecc.size; /* ECC chunk size */ 1592 int meta, n, page_size; 1593 unsigned int max_bitflips; 1594 unsigned int ecc_strength; 1595 int first, last, marker_pos; 1596 int ecc_parity_size; 1597 int col = 0; 1598 int ret; 1599 1600 /* The size of ECC parity */ 1601 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8; 1602 1603 /* Align it with the chunk size */ 1604 first = offs / size; 1605 last = (offs + len - 1) / size; 1606 1607 if (this->swap_block_mark) { 1608 /* 1609 * Find the chunk which contains the Block Marker. 1610 * If this chunk is in the range of [first, last], 1611 * we have to read out the whole page. 1612 * Why? since we had swapped the data at the position of Block 1613 * Marker to the metadata which is bound with the chunk 0. 1614 */ 1615 marker_pos = geo->block_mark_byte_offset / size; 1616 if (last >= marker_pos && first <= marker_pos) { 1617 dev_dbg(this->dev, 1618 "page:%d, first:%d, last:%d, marker at:%d\n", 1619 page, first, last, marker_pos); 1620 return gpmi_ecc_read_page(chip, buf, 0, page); 1621 } 1622 } 1623 1624 /* 1625 * if there is an ECC dedicate for meta: 1626 * - need to add an extra ECC size when calculating col and page_size, 1627 * if the meta size is NOT zero. 1628 * - ecc0_chunk size need to set to the same size as other chunks, 1629 * if the meta size is zero. 1630 */ 1631 1632 meta = geo->metadata_size; 1633 if (first) { 1634 if (geo->ecc_for_meta) 1635 col = meta + ecc_parity_size 1636 + (size + ecc_parity_size) * first; 1637 else 1638 col = meta + (size + ecc_parity_size) * first; 1639 1640 meta = 0; 1641 buf = buf + first * size; 1642 } 1643 1644 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8; 1645 n = last - first + 1; 1646 1647 if (geo->ecc_for_meta && meta) 1648 page_size = meta + ecc_parity_size 1649 + (size + ecc_parity_size) * n; 1650 else 1651 page_size = meta + (size + ecc_parity_size) * n; 1652 1653 ecc_strength = geo->ecc_strength >> 1; 1654 1655 this->bch_flashlayout0 = BF_BCH_FLASH0LAYOUT0_NBLOCKS( 1656 (geo->ecc_for_meta ? n : n - 1)) | 1657 BF_BCH_FLASH0LAYOUT0_META_SIZE(meta) | 1658 BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) | 1659 BF_BCH_FLASH0LAYOUT0_GF(geo->gf_len, this) | 1660 BF_BCH_FLASH0LAYOUT0_DATA0_SIZE((geo->ecc_for_meta ? 1661 0 : geo->ecc0_chunk_size), this); 1662 1663 this->bch_flashlayout1 = BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size) | 1664 BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) | 1665 BF_BCH_FLASH0LAYOUT1_GF(geo->gf_len, this) | 1666 BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(geo->eccn_chunk_size, this); 1667 1668 this->bch = true; 1669 1670 ret = nand_read_page_op(chip, page, col, buf, page_size); 1671 if (ret) 1672 return ret; 1673 1674 dev_dbg(this->dev, "page:%d(%d:%d)%d, chunk:(%d:%d), BCH PG size:%d\n", 1675 page, offs, len, col, first, n, page_size); 1676 1677 max_bitflips = gpmi_count_bitflips(chip, buf, first, last, meta); 1678 1679 return max_bitflips; 1680} 1681 1682static int gpmi_ecc_write_page(struct nand_chip *chip, const uint8_t *buf, 1683 int oob_required, int page) 1684{ 1685 struct mtd_info *mtd = nand_to_mtd(chip); 1686 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1687 struct bch_geometry *nfc_geo = &this->bch_geometry; 1688 1689 dev_dbg(this->dev, "ecc write page.\n"); 1690 1691 gpmi_bch_layout_std(this); 1692 this->bch = true; 1693 1694 memcpy(this->auxiliary_virt, chip->oob_poi, nfc_geo->auxiliary_size); 1695 1696 if (this->swap_block_mark) { 1697 /* 1698 * When doing bad block marker swapping we must always copy the 1699 * input buffer as we can't modify the const buffer. 1700 */ 1701 memcpy(this->data_buffer_dma, buf, mtd->writesize); 1702 buf = this->data_buffer_dma; 1703 block_mark_swapping(this, this->data_buffer_dma, 1704 this->auxiliary_virt); 1705 } 1706 1707 return nand_prog_page_op(chip, page, 0, buf, nfc_geo->page_size); 1708} 1709 1710/* 1711 * There are several places in this driver where we have to handle the OOB and 1712 * block marks. This is the function where things are the most complicated, so 1713 * this is where we try to explain it all. All the other places refer back to 1714 * here. 1715 * 1716 * These are the rules, in order of decreasing importance: 1717 * 1718 * 1) Nothing the caller does can be allowed to imperil the block mark. 1719 * 1720 * 2) In read operations, the first byte of the OOB we return must reflect the 1721 * true state of the block mark, no matter where that block mark appears in 1722 * the physical page. 1723 * 1724 * 3) ECC-based read operations return an OOB full of set bits (since we never 1725 * allow ECC-based writes to the OOB, it doesn't matter what ECC-based reads 1726 * return). 1727 * 1728 * 4) "Raw" read operations return a direct view of the physical bytes in the 1729 * page, using the conventional definition of which bytes are data and which 1730 * are OOB. This gives the caller a way to see the actual, physical bytes 1731 * in the page, without the distortions applied by our ECC engine. 1732 * 1733 * 1734 * What we do for this specific read operation depends on two questions: 1735 * 1736 * 1) Are we doing a "raw" read, or an ECC-based read? 1737 * 1738 * 2) Are we using block mark swapping or transcription? 1739 * 1740 * There are four cases, illustrated by the following Karnaugh map: 1741 * 1742 * | Raw | ECC-based | 1743 * -------------+-------------------------+-------------------------+ 1744 * | Read the conventional | | 1745 * | OOB at the end of the | | 1746 * Swapping | page and return it. It | | 1747 * | contains exactly what | | 1748 * | we want. | Read the block mark and | 1749 * -------------+-------------------------+ return it in a buffer | 1750 * | Read the conventional | full of set bits. | 1751 * | OOB at the end of the | | 1752 * | page and also the block | | 1753 * Transcribing | mark in the metadata. | | 1754 * | Copy the block mark | | 1755 * | into the first byte of | | 1756 * | the OOB. | | 1757 * -------------+-------------------------+-------------------------+ 1758 * 1759 * Note that we break rule #4 in the Transcribing/Raw case because we're not 1760 * giving an accurate view of the actual, physical bytes in the page (we're 1761 * overwriting the block mark). That's OK because it's more important to follow 1762 * rule #2. 1763 * 1764 * It turns out that knowing whether we want an "ECC-based" or "raw" read is not 1765 * easy. When reading a page, for example, the NAND Flash MTD code calls our 1766 * ecc.read_page or ecc.read_page_raw function. Thus, the fact that MTD wants an 1767 * ECC-based or raw view of the page is implicit in which function it calls 1768 * (there is a similar pair of ECC-based/raw functions for writing). 1769 */ 1770static int gpmi_ecc_read_oob(struct nand_chip *chip, int page) 1771{ 1772 struct mtd_info *mtd = nand_to_mtd(chip); 1773 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1774 int ret; 1775 1776 /* clear the OOB buffer */ 1777 memset(chip->oob_poi, ~0, mtd->oobsize); 1778 1779 /* Read out the conventional OOB. */ 1780 ret = nand_read_page_op(chip, page, mtd->writesize, chip->oob_poi, 1781 mtd->oobsize); 1782 if (ret) 1783 return ret; 1784 1785 /* 1786 * Now, we want to make sure the block mark is correct. In the 1787 * non-transcribing case (!GPMI_IS_MX23()), we already have it. 1788 * Otherwise, we need to explicitly read it. 1789 */ 1790 if (GPMI_IS_MX23(this)) { 1791 /* Read the block mark into the first byte of the OOB buffer. */ 1792 ret = nand_read_page_op(chip, page, 0, chip->oob_poi, 1); 1793 if (ret) 1794 return ret; 1795 } 1796 1797 return 0; 1798} 1799 1800static int gpmi_ecc_write_oob(struct nand_chip *chip, int page) 1801{ 1802 struct mtd_info *mtd = nand_to_mtd(chip); 1803 struct mtd_oob_region of = { }; 1804 1805 /* Do we have available oob area? */ 1806 mtd_ooblayout_free(mtd, 0, &of); 1807 if (!of.length) 1808 return -EPERM; 1809 1810 if (!nand_is_slc(chip)) 1811 return -EPERM; 1812 1813 return nand_prog_page_op(chip, page, mtd->writesize + of.offset, 1814 chip->oob_poi + of.offset, of.length); 1815} 1816 1817/* 1818 * This function reads a NAND page without involving the ECC engine (no HW 1819 * ECC correction). 1820 * The tricky part in the GPMI/BCH controller is that it stores ECC bits 1821 * inline (interleaved with payload DATA), and do not align data chunk on 1822 * byte boundaries. 1823 * We thus need to take care moving the payload data and ECC bits stored in the 1824 * page into the provided buffers, which is why we're using nand_extract_bits(). 1825 * 1826 * See set_geometry_by_ecc_info inline comments to have a full description 1827 * of the layout used by the GPMI controller. 1828 */ 1829static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf, 1830 int oob_required, int page) 1831{ 1832 struct mtd_info *mtd = nand_to_mtd(chip); 1833 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1834 struct bch_geometry *nfc_geo = &this->bch_geometry; 1835 int eccsize = nfc_geo->eccn_chunk_size; 1836 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len; 1837 u8 *tmp_buf = this->raw_buffer; 1838 size_t src_bit_off; 1839 size_t oob_bit_off; 1840 size_t oob_byte_off; 1841 uint8_t *oob = chip->oob_poi; 1842 int step; 1843 int ret; 1844 1845 ret = nand_read_page_op(chip, page, 0, tmp_buf, 1846 mtd->writesize + mtd->oobsize); 1847 if (ret) 1848 return ret; 1849 1850 /* 1851 * If required, swap the bad block marker and the data stored in the 1852 * metadata section, so that we don't wrongly consider a block as bad. 1853 * 1854 * See the layout description for a detailed explanation on why this 1855 * is needed. 1856 */ 1857 if (this->swap_block_mark) 1858 swap(tmp_buf[0], tmp_buf[mtd->writesize]); 1859 1860 /* 1861 * Copy the metadata section into the oob buffer (this section is 1862 * guaranteed to be aligned on a byte boundary). 1863 */ 1864 if (oob_required) 1865 memcpy(oob, tmp_buf, nfc_geo->metadata_size); 1866 1867 oob_bit_off = nfc_geo->metadata_size * 8; 1868 src_bit_off = oob_bit_off; 1869 1870 /* Extract interleaved payload data and ECC bits */ 1871 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) { 1872 if (buf) 1873 nand_extract_bits(buf, step * eccsize * 8, tmp_buf, 1874 src_bit_off, eccsize * 8); 1875 src_bit_off += eccsize * 8; 1876 1877 /* Align last ECC block to align a byte boundary */ 1878 if (step == nfc_geo->ecc_chunk_count - 1 && 1879 (oob_bit_off + eccbits) % 8) 1880 eccbits += 8 - ((oob_bit_off + eccbits) % 8); 1881 1882 if (oob_required) 1883 nand_extract_bits(oob, oob_bit_off, tmp_buf, 1884 src_bit_off, eccbits); 1885 1886 src_bit_off += eccbits; 1887 oob_bit_off += eccbits; 1888 } 1889 1890 if (oob_required) { 1891 oob_byte_off = oob_bit_off / 8; 1892 1893 if (oob_byte_off < mtd->oobsize) 1894 memcpy(oob + oob_byte_off, 1895 tmp_buf + mtd->writesize + oob_byte_off, 1896 mtd->oobsize - oob_byte_off); 1897 } 1898 1899 return 0; 1900} 1901 1902/* 1903 * This function writes a NAND page without involving the ECC engine (no HW 1904 * ECC generation). 1905 * The tricky part in the GPMI/BCH controller is that it stores ECC bits 1906 * inline (interleaved with payload DATA), and do not align data chunk on 1907 * byte boundaries. 1908 * We thus need to take care moving the OOB area at the right place in the 1909 * final page, which is why we're using nand_extract_bits(). 1910 * 1911 * See set_geometry_by_ecc_info inline comments to have a full description 1912 * of the layout used by the GPMI controller. 1913 */ 1914static int gpmi_ecc_write_page_raw(struct nand_chip *chip, const uint8_t *buf, 1915 int oob_required, int page) 1916{ 1917 struct mtd_info *mtd = nand_to_mtd(chip); 1918 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1919 struct bch_geometry *nfc_geo = &this->bch_geometry; 1920 int eccsize = nfc_geo->eccn_chunk_size; 1921 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len; 1922 u8 *tmp_buf = this->raw_buffer; 1923 uint8_t *oob = chip->oob_poi; 1924 size_t dst_bit_off; 1925 size_t oob_bit_off; 1926 size_t oob_byte_off; 1927 int step; 1928 1929 /* 1930 * Initialize all bits to 1 in case we don't have a buffer for the 1931 * payload or oob data in order to leave unspecified bits of data 1932 * to their initial state. 1933 */ 1934 if (!buf || !oob_required) 1935 memset(tmp_buf, 0xff, mtd->writesize + mtd->oobsize); 1936 1937 /* 1938 * First copy the metadata section (stored in oob buffer) at the 1939 * beginning of the page, as imposed by the GPMI layout. 1940 */ 1941 memcpy(tmp_buf, oob, nfc_geo->metadata_size); 1942 oob_bit_off = nfc_geo->metadata_size * 8; 1943 dst_bit_off = oob_bit_off; 1944 1945 /* Interleave payload data and ECC bits */ 1946 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) { 1947 if (buf) 1948 nand_extract_bits(tmp_buf, dst_bit_off, buf, 1949 step * eccsize * 8, eccsize * 8); 1950 dst_bit_off += eccsize * 8; 1951 1952 /* Align last ECC block to align a byte boundary */ 1953 if (step == nfc_geo->ecc_chunk_count - 1 && 1954 (oob_bit_off + eccbits) % 8) 1955 eccbits += 8 - ((oob_bit_off + eccbits) % 8); 1956 1957 if (oob_required) 1958 nand_extract_bits(tmp_buf, dst_bit_off, oob, 1959 oob_bit_off, eccbits); 1960 1961 dst_bit_off += eccbits; 1962 oob_bit_off += eccbits; 1963 } 1964 1965 oob_byte_off = oob_bit_off / 8; 1966 1967 if (oob_required && oob_byte_off < mtd->oobsize) 1968 memcpy(tmp_buf + mtd->writesize + oob_byte_off, 1969 oob + oob_byte_off, mtd->oobsize - oob_byte_off); 1970 1971 /* 1972 * If required, swap the bad block marker and the first byte of the 1973 * metadata section, so that we don't modify the bad block marker. 1974 * 1975 * See the layout description for a detailed explanation on why this 1976 * is needed. 1977 */ 1978 if (this->swap_block_mark) 1979 swap(tmp_buf[0], tmp_buf[mtd->writesize]); 1980 1981 return nand_prog_page_op(chip, page, 0, tmp_buf, 1982 mtd->writesize + mtd->oobsize); 1983} 1984 1985static int gpmi_ecc_read_oob_raw(struct nand_chip *chip, int page) 1986{ 1987 return gpmi_ecc_read_page_raw(chip, NULL, 1, page); 1988} 1989 1990static int gpmi_ecc_write_oob_raw(struct nand_chip *chip, int page) 1991{ 1992 return gpmi_ecc_write_page_raw(chip, NULL, 1, page); 1993} 1994 1995static int gpmi_block_markbad(struct nand_chip *chip, loff_t ofs) 1996{ 1997 struct mtd_info *mtd = nand_to_mtd(chip); 1998 struct gpmi_nand_data *this = nand_get_controller_data(chip); 1999 int ret = 0; 2000 uint8_t *block_mark; 2001 int column, page, chipnr; 2002 2003 chipnr = (int)(ofs >> chip->chip_shift); 2004 nand_select_target(chip, chipnr); 2005 2006 column = !GPMI_IS_MX23(this) ? mtd->writesize : 0; 2007 2008 /* Write the block mark. */ 2009 block_mark = this->data_buffer_dma; 2010 block_mark[0] = 0; /* bad block marker */ 2011 2012 /* Shift to get page */ 2013 page = (int)(ofs >> chip->page_shift); 2014 2015 ret = nand_prog_page_op(chip, page, column, block_mark, 1); 2016 2017 nand_deselect_target(chip); 2018 2019 return ret; 2020} 2021 2022static int nand_boot_set_geometry(struct gpmi_nand_data *this) 2023{ 2024 struct boot_rom_geometry *geometry = &this->rom_geometry; 2025 2026 /* 2027 * Set the boot block stride size. 2028 * 2029 * In principle, we should be reading this from the OTP bits, since 2030 * that's where the ROM is going to get it. In fact, we don't have any 2031 * way to read the OTP bits, so we go with the default and hope for the 2032 * best. 2033 */ 2034 geometry->stride_size_in_pages = 64; 2035 2036 /* 2037 * Set the search area stride exponent. 2038 * 2039 * In principle, we should be reading this from the OTP bits, since 2040 * that's where the ROM is going to get it. In fact, we don't have any 2041 * way to read the OTP bits, so we go with the default and hope for the 2042 * best. 2043 */ 2044 geometry->search_area_stride_exponent = 2; 2045 return 0; 2046} 2047 2048static const char *fingerprint = "STMP"; 2049static int mx23_check_transcription_stamp(struct gpmi_nand_data *this) 2050{ 2051 struct boot_rom_geometry *rom_geo = &this->rom_geometry; 2052 struct device *dev = this->dev; 2053 struct nand_chip *chip = &this->nand; 2054 unsigned int search_area_size_in_strides; 2055 unsigned int stride; 2056 unsigned int page; 2057 u8 *buffer = nand_get_data_buf(chip); 2058 int found_an_ncb_fingerprint = false; 2059 int ret; 2060 2061 /* Compute the number of strides in a search area. */ 2062 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent; 2063 2064 nand_select_target(chip, 0); 2065 2066 /* 2067 * Loop through the first search area, looking for the NCB fingerprint. 2068 */ 2069 dev_dbg(dev, "Scanning for an NCB fingerprint...\n"); 2070 2071 for (stride = 0; stride < search_area_size_in_strides; stride++) { 2072 /* Compute the page addresses. */ 2073 page = stride * rom_geo->stride_size_in_pages; 2074 2075 dev_dbg(dev, "Looking for a fingerprint in page 0x%x\n", page); 2076 2077 /* 2078 * Read the NCB fingerprint. The fingerprint is four bytes long 2079 * and starts in the 12th byte of the page. 2080 */ 2081 ret = nand_read_page_op(chip, page, 12, buffer, 2082 strlen(fingerprint)); 2083 if (ret) 2084 continue; 2085 2086 /* Look for the fingerprint. */ 2087 if (!memcmp(buffer, fingerprint, strlen(fingerprint))) { 2088 found_an_ncb_fingerprint = true; 2089 break; 2090 } 2091 2092 } 2093 2094 nand_deselect_target(chip); 2095 2096 if (found_an_ncb_fingerprint) 2097 dev_dbg(dev, "\tFound a fingerprint\n"); 2098 else 2099 dev_dbg(dev, "\tNo fingerprint found\n"); 2100 return found_an_ncb_fingerprint; 2101} 2102 2103/* Writes a transcription stamp. */ 2104static int mx23_write_transcription_stamp(struct gpmi_nand_data *this) 2105{ 2106 struct device *dev = this->dev; 2107 struct boot_rom_geometry *rom_geo = &this->rom_geometry; 2108 struct nand_chip *chip = &this->nand; 2109 struct mtd_info *mtd = nand_to_mtd(chip); 2110 unsigned int block_size_in_pages; 2111 unsigned int search_area_size_in_strides; 2112 unsigned int search_area_size_in_pages; 2113 unsigned int search_area_size_in_blocks; 2114 unsigned int block; 2115 unsigned int stride; 2116 unsigned int page; 2117 u8 *buffer = nand_get_data_buf(chip); 2118 int status; 2119 2120 /* Compute the search area geometry. */ 2121 block_size_in_pages = mtd->erasesize / mtd->writesize; 2122 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent; 2123 search_area_size_in_pages = search_area_size_in_strides * 2124 rom_geo->stride_size_in_pages; 2125 search_area_size_in_blocks = 2126 (search_area_size_in_pages + (block_size_in_pages - 1)) / 2127 block_size_in_pages; 2128 2129 dev_dbg(dev, "Search Area Geometry :\n"); 2130 dev_dbg(dev, "\tin Blocks : %u\n", search_area_size_in_blocks); 2131 dev_dbg(dev, "\tin Strides: %u\n", search_area_size_in_strides); 2132 dev_dbg(dev, "\tin Pages : %u\n", search_area_size_in_pages); 2133 2134 nand_select_target(chip, 0); 2135 2136 /* Loop over blocks in the first search area, erasing them. */ 2137 dev_dbg(dev, "Erasing the search area...\n"); 2138 2139 for (block = 0; block < search_area_size_in_blocks; block++) { 2140 /* Erase this block. */ 2141 dev_dbg(dev, "\tErasing block 0x%x\n", block); 2142 status = nand_erase_op(chip, block); 2143 if (status) 2144 dev_err(dev, "[%s] Erase failed.\n", __func__); 2145 } 2146 2147 /* Write the NCB fingerprint into the page buffer. */ 2148 memset(buffer, ~0, mtd->writesize); 2149 memcpy(buffer + 12, fingerprint, strlen(fingerprint)); 2150 2151 /* Loop through the first search area, writing NCB fingerprints. */ 2152 dev_dbg(dev, "Writing NCB fingerprints...\n"); 2153 for (stride = 0; stride < search_area_size_in_strides; stride++) { 2154 /* Compute the page addresses. */ 2155 page = stride * rom_geo->stride_size_in_pages; 2156 2157 /* Write the first page of the current stride. */ 2158 dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page); 2159 2160 status = chip->ecc.write_page_raw(chip, buffer, 0, page); 2161 if (status) 2162 dev_err(dev, "[%s] Write failed.\n", __func__); 2163 } 2164 2165 nand_deselect_target(chip); 2166 2167 return 0; 2168} 2169 2170static int mx23_boot_init(struct gpmi_nand_data *this) 2171{ 2172 struct device *dev = this->dev; 2173 struct nand_chip *chip = &this->nand; 2174 struct mtd_info *mtd = nand_to_mtd(chip); 2175 unsigned int block_count; 2176 unsigned int block; 2177 int chipnr; 2178 int page; 2179 loff_t byte; 2180 uint8_t block_mark; 2181 int ret = 0; 2182 2183 /* 2184 * If control arrives here, we can't use block mark swapping, which 2185 * means we're forced to use transcription. First, scan for the 2186 * transcription stamp. If we find it, then we don't have to do 2187 * anything -- the block marks are already transcribed. 2188 */ 2189 if (mx23_check_transcription_stamp(this)) 2190 return 0; 2191 2192 /* 2193 * If control arrives here, we couldn't find a transcription stamp, so 2194 * so we presume the block marks are in the conventional location. 2195 */ 2196 dev_dbg(dev, "Transcribing bad block marks...\n"); 2197 2198 /* Compute the number of blocks in the entire medium. */ 2199 block_count = nanddev_eraseblocks_per_target(&chip->base); 2200 2201 /* 2202 * Loop over all the blocks in the medium, transcribing block marks as 2203 * we go. 2204 */ 2205 for (block = 0; block < block_count; block++) { 2206 /* 2207 * Compute the chip, page and byte addresses for this block's 2208 * conventional mark. 2209 */ 2210 chipnr = block >> (chip->chip_shift - chip->phys_erase_shift); 2211 page = block << (chip->phys_erase_shift - chip->page_shift); 2212 byte = block << chip->phys_erase_shift; 2213 2214 /* Send the command to read the conventional block mark. */ 2215 nand_select_target(chip, chipnr); 2216 ret = nand_read_page_op(chip, page, mtd->writesize, &block_mark, 2217 1); 2218 nand_deselect_target(chip); 2219 2220 if (ret) 2221 continue; 2222 2223 /* 2224 * Check if the block is marked bad. If so, we need to mark it 2225 * again, but this time the result will be a mark in the 2226 * location where we transcribe block marks. 2227 */ 2228 if (block_mark != 0xff) { 2229 dev_dbg(dev, "Transcribing mark in block %u\n", block); 2230 ret = chip->legacy.block_markbad(chip, byte); 2231 if (ret) 2232 dev_err(dev, 2233 "Failed to mark block bad with ret %d\n", 2234 ret); 2235 } 2236 } 2237 2238 /* Write the stamp that indicates we've transcribed the block marks. */ 2239 mx23_write_transcription_stamp(this); 2240 return 0; 2241} 2242 2243static int nand_boot_init(struct gpmi_nand_data *this) 2244{ 2245 nand_boot_set_geometry(this); 2246 2247 /* This is ROM arch-specific initilization before the BBT scanning. */ 2248 if (GPMI_IS_MX23(this)) 2249 return mx23_boot_init(this); 2250 return 0; 2251} 2252 2253static int gpmi_set_geometry(struct gpmi_nand_data *this) 2254{ 2255 int ret; 2256 2257 /* Free the temporary DMA memory for reading ID. */ 2258 gpmi_free_dma_buffer(this); 2259 2260 /* Set up the NFC geometry which is used by BCH. */ 2261 ret = bch_set_geometry(this); 2262 if (ret) { 2263 dev_err(this->dev, "Error setting BCH geometry : %d\n", ret); 2264 return ret; 2265 } 2266 2267 /* Alloc the new DMA buffers according to the pagesize and oobsize */ 2268 return gpmi_alloc_dma_buffer(this); 2269} 2270 2271static int gpmi_init_last(struct gpmi_nand_data *this) 2272{ 2273 struct nand_chip *chip = &this->nand; 2274 struct mtd_info *mtd = nand_to_mtd(chip); 2275 struct nand_ecc_ctrl *ecc = &chip->ecc; 2276 struct bch_geometry *bch_geo = &this->bch_geometry; 2277 int ret; 2278 2279 /* Set up the medium geometry */ 2280 ret = gpmi_set_geometry(this); 2281 if (ret) 2282 return ret; 2283 2284 /* Init the nand_ecc_ctrl{} */ 2285 ecc->read_page = gpmi_ecc_read_page; 2286 ecc->write_page = gpmi_ecc_write_page; 2287 ecc->read_oob = gpmi_ecc_read_oob; 2288 ecc->write_oob = gpmi_ecc_write_oob; 2289 ecc->read_page_raw = gpmi_ecc_read_page_raw; 2290 ecc->write_page_raw = gpmi_ecc_write_page_raw; 2291 ecc->read_oob_raw = gpmi_ecc_read_oob_raw; 2292 ecc->write_oob_raw = gpmi_ecc_write_oob_raw; 2293 ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2294 ecc->size = bch_geo->eccn_chunk_size; 2295 ecc->strength = bch_geo->ecc_strength; 2296 mtd_set_ooblayout(mtd, &gpmi_ooblayout_ops); 2297 2298 /* 2299 * We only enable the subpage read when: 2300 * (1) the chip is imx6, and 2301 * (2) the size of the ECC parity is byte aligned. 2302 */ 2303 if (GPMI_IS_MX6(this) && 2304 ((bch_geo->gf_len * bch_geo->ecc_strength) % 8) == 0) { 2305 ecc->read_subpage = gpmi_ecc_read_subpage; 2306 chip->options |= NAND_SUBPAGE_READ; 2307 } 2308 2309 return 0; 2310} 2311 2312static int gpmi_nand_attach_chip(struct nand_chip *chip) 2313{ 2314 struct gpmi_nand_data *this = nand_get_controller_data(chip); 2315 int ret; 2316 2317 if (chip->bbt_options & NAND_BBT_USE_FLASH) { 2318 chip->bbt_options |= NAND_BBT_NO_OOB; 2319 2320 if (of_property_read_bool(this->dev->of_node, 2321 "fsl,no-blockmark-swap")) 2322 this->swap_block_mark = false; 2323 } 2324 dev_dbg(this->dev, "Blockmark swapping %s\n", 2325 str_enabled_disabled(this->swap_block_mark)); 2326 2327 ret = gpmi_init_last(this); 2328 if (ret) 2329 return ret; 2330 2331 chip->options |= NAND_SKIP_BBTSCAN; 2332 2333 return 0; 2334} 2335 2336static struct gpmi_transfer *get_next_transfer(struct gpmi_nand_data *this) 2337{ 2338 struct gpmi_transfer *transfer = &this->transfers[this->ntransfers]; 2339 2340 this->ntransfers++; 2341 2342 if (this->ntransfers == GPMI_MAX_TRANSFERS) 2343 return NULL; 2344 2345 return transfer; 2346} 2347 2348static struct dma_async_tx_descriptor *gpmi_chain_command( 2349 struct gpmi_nand_data *this, u8 cmd, const u8 *addr, int naddr) 2350{ 2351 struct dma_chan *channel = get_dma_chan(this); 2352 struct dma_async_tx_descriptor *desc; 2353 struct gpmi_transfer *transfer; 2354 int chip = this->nand.cur_cs; 2355 u32 pio[3]; 2356 2357 /* [1] send out the PIO words */ 2358 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE) 2359 | BM_GPMI_CTRL0_WORD_LENGTH 2360 | BF_GPMI_CTRL0_CS(chip, this) 2361 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2362 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_CLE) 2363 | BM_GPMI_CTRL0_ADDRESS_INCREMENT 2364 | BF_GPMI_CTRL0_XFER_COUNT(naddr + 1); 2365 pio[1] = 0; 2366 pio[2] = 0; 2367 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio), 2368 DMA_TRANS_NONE, 0); 2369 if (!desc) 2370 return NULL; 2371 2372 transfer = get_next_transfer(this); 2373 if (!transfer) 2374 return NULL; 2375 2376 transfer->cmdbuf[0] = cmd; 2377 if (naddr) 2378 memcpy(&transfer->cmdbuf[1], addr, naddr); 2379 2380 sg_init_one(&transfer->sgl, transfer->cmdbuf, naddr + 1); 2381 dma_map_sg(this->dev, &transfer->sgl, 1, DMA_TO_DEVICE); 2382 2383 transfer->direction = DMA_TO_DEVICE; 2384 2385 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, DMA_MEM_TO_DEV, 2386 MXS_DMA_CTRL_WAIT4END); 2387 return desc; 2388} 2389 2390static struct dma_async_tx_descriptor *gpmi_chain_wait_ready( 2391 struct gpmi_nand_data *this) 2392{ 2393 struct dma_chan *channel = get_dma_chan(this); 2394 u32 pio[2]; 2395 2396 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY) 2397 | BM_GPMI_CTRL0_WORD_LENGTH 2398 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this) 2399 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2400 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA) 2401 | BF_GPMI_CTRL0_XFER_COUNT(0); 2402 pio[1] = 0; 2403 2404 return mxs_dmaengine_prep_pio(channel, pio, 2, DMA_TRANS_NONE, 2405 MXS_DMA_CTRL_WAIT4END | MXS_DMA_CTRL_WAIT4RDY); 2406} 2407 2408static struct dma_async_tx_descriptor *gpmi_chain_data_read( 2409 struct gpmi_nand_data *this, void *buf, int raw_len, bool *direct) 2410{ 2411 struct dma_async_tx_descriptor *desc; 2412 struct dma_chan *channel = get_dma_chan(this); 2413 struct gpmi_transfer *transfer; 2414 u32 pio[6] = {}; 2415 2416 transfer = get_next_transfer(this); 2417 if (!transfer) 2418 return NULL; 2419 2420 transfer->direction = DMA_FROM_DEVICE; 2421 2422 *direct = prepare_data_dma(this, buf, raw_len, &transfer->sgl, 2423 DMA_FROM_DEVICE); 2424 2425 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__READ) 2426 | BM_GPMI_CTRL0_WORD_LENGTH 2427 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this) 2428 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2429 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA) 2430 | BF_GPMI_CTRL0_XFER_COUNT(raw_len); 2431 2432 if (this->bch) { 2433 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC 2434 | BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_DECODE) 2435 | BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE 2436 | BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY); 2437 pio[3] = raw_len; 2438 pio[4] = transfer->sgl.dma_address; 2439 pio[5] = this->auxiliary_phys; 2440 } 2441 2442 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio), 2443 DMA_TRANS_NONE, 0); 2444 if (!desc) 2445 return NULL; 2446 2447 if (!this->bch) 2448 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, 2449 DMA_DEV_TO_MEM, 2450 MXS_DMA_CTRL_WAIT4END); 2451 2452 return desc; 2453} 2454 2455static struct dma_async_tx_descriptor *gpmi_chain_data_write( 2456 struct gpmi_nand_data *this, const void *buf, int raw_len) 2457{ 2458 struct dma_chan *channel = get_dma_chan(this); 2459 struct dma_async_tx_descriptor *desc; 2460 struct gpmi_transfer *transfer; 2461 u32 pio[6] = {}; 2462 2463 transfer = get_next_transfer(this); 2464 if (!transfer) 2465 return NULL; 2466 2467 transfer->direction = DMA_TO_DEVICE; 2468 2469 prepare_data_dma(this, buf, raw_len, &transfer->sgl, DMA_TO_DEVICE); 2470 2471 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE) 2472 | BM_GPMI_CTRL0_WORD_LENGTH 2473 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this) 2474 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this) 2475 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA) 2476 | BF_GPMI_CTRL0_XFER_COUNT(raw_len); 2477 2478 if (this->bch) { 2479 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC 2480 | BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_ENCODE) 2481 | BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE | 2482 BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY); 2483 pio[3] = raw_len; 2484 pio[4] = transfer->sgl.dma_address; 2485 pio[5] = this->auxiliary_phys; 2486 } 2487 2488 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio), 2489 DMA_TRANS_NONE, 2490 (this->bch ? MXS_DMA_CTRL_WAIT4END : 0)); 2491 if (!desc) 2492 return NULL; 2493 2494 if (!this->bch) 2495 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, 2496 DMA_MEM_TO_DEV, 2497 MXS_DMA_CTRL_WAIT4END); 2498 2499 return desc; 2500} 2501 2502static int gpmi_nfc_exec_op(struct nand_chip *chip, 2503 const struct nand_operation *op, 2504 bool check_only) 2505{ 2506 const struct nand_op_instr *instr; 2507 struct gpmi_nand_data *this = nand_get_controller_data(chip); 2508 struct dma_async_tx_descriptor *desc = NULL; 2509 int i, ret, buf_len = 0, nbufs = 0; 2510 u8 cmd = 0; 2511 void *buf_read = NULL; 2512 const void *buf_write = NULL; 2513 bool direct = false; 2514 struct completion *dma_completion, *bch_completion; 2515 unsigned long to; 2516 2517 if (check_only) 2518 return 0; 2519 2520 this->ntransfers = 0; 2521 for (i = 0; i < GPMI_MAX_TRANSFERS; i++) 2522 this->transfers[i].direction = DMA_NONE; 2523 2524 ret = pm_runtime_resume_and_get(this->dev); 2525 if (ret < 0) 2526 return ret; 2527 2528 /* 2529 * This driver currently supports only one NAND chip. Plus, dies share 2530 * the same configuration. So once timings have been applied on the 2531 * controller side, they will not change anymore. When the time will 2532 * come, the check on must_apply_timings will have to be dropped. 2533 */ 2534 if (this->hw.must_apply_timings) { 2535 this->hw.must_apply_timings = false; 2536 ret = gpmi_nfc_apply_timings(this); 2537 if (ret) 2538 goto out_pm; 2539 } 2540 2541 dev_dbg(this->dev, "%s: %d instructions\n", __func__, op->ninstrs); 2542 2543 for (i = 0; i < op->ninstrs; i++) { 2544 instr = &op->instrs[i]; 2545 2546 nand_op_trace(" ", instr); 2547 2548 switch (instr->type) { 2549 case NAND_OP_WAITRDY_INSTR: 2550 desc = gpmi_chain_wait_ready(this); 2551 break; 2552 case NAND_OP_CMD_INSTR: 2553 cmd = instr->ctx.cmd.opcode; 2554 2555 /* 2556 * When this command has an address cycle chain it 2557 * together with the address cycle 2558 */ 2559 if (i + 1 != op->ninstrs && 2560 op->instrs[i + 1].type == NAND_OP_ADDR_INSTR) 2561 continue; 2562 2563 desc = gpmi_chain_command(this, cmd, NULL, 0); 2564 2565 break; 2566 case NAND_OP_ADDR_INSTR: 2567 desc = gpmi_chain_command(this, cmd, instr->ctx.addr.addrs, 2568 instr->ctx.addr.naddrs); 2569 break; 2570 case NAND_OP_DATA_OUT_INSTR: 2571 buf_write = instr->ctx.data.buf.out; 2572 buf_len = instr->ctx.data.len; 2573 nbufs++; 2574 2575 desc = gpmi_chain_data_write(this, buf_write, buf_len); 2576 2577 break; 2578 case NAND_OP_DATA_IN_INSTR: 2579 if (!instr->ctx.data.len) 2580 break; 2581 buf_read = instr->ctx.data.buf.in; 2582 buf_len = instr->ctx.data.len; 2583 nbufs++; 2584 2585 desc = gpmi_chain_data_read(this, buf_read, buf_len, 2586 &direct); 2587 break; 2588 } 2589 2590 if (!desc) { 2591 ret = -ENXIO; 2592 goto unmap; 2593 } 2594 } 2595 2596 dev_dbg(this->dev, "%s setup done\n", __func__); 2597 2598 if (nbufs > 1) { 2599 dev_err(this->dev, "Multiple data instructions not supported\n"); 2600 ret = -EINVAL; 2601 goto unmap; 2602 } 2603 2604 if (this->bch) { 2605 writel(this->bch_flashlayout0, 2606 this->resources.bch_regs + HW_BCH_FLASH0LAYOUT0); 2607 writel(this->bch_flashlayout1, 2608 this->resources.bch_regs + HW_BCH_FLASH0LAYOUT1); 2609 } 2610 2611 desc->callback = dma_irq_callback; 2612 desc->callback_param = this; 2613 dma_completion = &this->dma_done; 2614 bch_completion = NULL; 2615 2616 init_completion(dma_completion); 2617 2618 if (this->bch && buf_read) { 2619 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN, 2620 this->resources.bch_regs + HW_BCH_CTRL_SET); 2621 bch_completion = &this->bch_done; 2622 init_completion(bch_completion); 2623 } 2624 2625 dmaengine_submit(desc); 2626 dma_async_issue_pending(get_dma_chan(this)); 2627 2628 to = wait_for_completion_timeout(dma_completion, msecs_to_jiffies(1000)); 2629 if (!to) { 2630 dev_err(this->dev, "DMA timeout, last DMA\n"); 2631 gpmi_dump_info(this); 2632 ret = -ETIMEDOUT; 2633 goto unmap; 2634 } 2635 2636 if (this->bch && buf_read) { 2637 to = wait_for_completion_timeout(bch_completion, msecs_to_jiffies(1000)); 2638 if (!to) { 2639 dev_err(this->dev, "BCH timeout, last DMA\n"); 2640 gpmi_dump_info(this); 2641 ret = -ETIMEDOUT; 2642 goto unmap; 2643 } 2644 } 2645 2646 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN, 2647 this->resources.bch_regs + HW_BCH_CTRL_CLR); 2648 gpmi_clear_bch(this); 2649 2650 ret = 0; 2651 2652unmap: 2653 for (i = 0; i < this->ntransfers; i++) { 2654 struct gpmi_transfer *transfer = &this->transfers[i]; 2655 2656 if (transfer->direction != DMA_NONE) 2657 dma_unmap_sg(this->dev, &transfer->sgl, 1, 2658 transfer->direction); 2659 } 2660 2661 if (!ret && buf_read && !direct) 2662 memcpy(buf_read, this->data_buffer_dma, 2663 gpmi_raw_len_to_len(this, buf_len)); 2664 2665 this->bch = false; 2666 2667out_pm: 2668 pm_runtime_put_autosuspend(this->dev); 2669 2670 return ret; 2671} 2672 2673static const struct nand_controller_ops gpmi_nand_controller_ops = { 2674 .attach_chip = gpmi_nand_attach_chip, 2675 .setup_interface = gpmi_setup_interface, 2676 .exec_op = gpmi_nfc_exec_op, 2677}; 2678 2679static int gpmi_nand_init(struct gpmi_nand_data *this) 2680{ 2681 struct nand_chip *chip = &this->nand; 2682 struct mtd_info *mtd = nand_to_mtd(chip); 2683 int ret; 2684 2685 /* init the MTD data structures */ 2686 mtd->name = "gpmi-nand"; 2687 mtd->dev.parent = this->dev; 2688 2689 /* init the nand_chip{}, we don't support a 16-bit NAND Flash bus. */ 2690 nand_set_controller_data(chip, this); 2691 nand_set_flash_node(chip, this->pdev->dev.of_node); 2692 chip->legacy.block_markbad = gpmi_block_markbad; 2693 chip->badblock_pattern = &gpmi_bbt_descr; 2694 chip->options |= NAND_NO_SUBPAGE_WRITE; 2695 2696 /* Set up swap_block_mark, must be set before the gpmi_set_geometry() */ 2697 this->swap_block_mark = !GPMI_IS_MX23(this); 2698 2699 /* 2700 * Allocate a temporary DMA buffer for reading ID in the 2701 * nand_scan_ident(). 2702 */ 2703 this->bch_geometry.payload_size = 1024; 2704 this->bch_geometry.auxiliary_size = 128; 2705 ret = gpmi_alloc_dma_buffer(this); 2706 if (ret) 2707 return ret; 2708 2709 nand_controller_init(&this->base); 2710 this->base.ops = &gpmi_nand_controller_ops; 2711 chip->controller = &this->base; 2712 2713 ret = nand_scan(chip, GPMI_IS_MX6(this) ? 2 : 1); 2714 if (ret) 2715 goto err_out; 2716 2717 ret = nand_boot_init(this); 2718 if (ret) 2719 goto err_nand_cleanup; 2720 ret = nand_create_bbt(chip); 2721 if (ret) 2722 goto err_nand_cleanup; 2723 2724 ret = mtd_device_register(mtd, NULL, 0); 2725 if (ret) 2726 goto err_nand_cleanup; 2727 return 0; 2728 2729err_nand_cleanup: 2730 nand_cleanup(chip); 2731err_out: 2732 gpmi_free_dma_buffer(this); 2733 return ret; 2734} 2735 2736static const struct of_device_id gpmi_nand_id_table[] = { 2737 { .compatible = "fsl,imx23-gpmi-nand", .data = &gpmi_devdata_imx23, }, 2738 { .compatible = "fsl,imx28-gpmi-nand", .data = &gpmi_devdata_imx28, }, 2739 { .compatible = "fsl,imx6q-gpmi-nand", .data = &gpmi_devdata_imx6q, }, 2740 { .compatible = "fsl,imx6sx-gpmi-nand", .data = &gpmi_devdata_imx6sx, }, 2741 { .compatible = "fsl,imx7d-gpmi-nand", .data = &gpmi_devdata_imx7d,}, 2742 { .compatible = "fsl,imx8qxp-gpmi-nand", .data = &gpmi_devdata_imx8qxp, }, 2743 {} 2744}; 2745MODULE_DEVICE_TABLE(of, gpmi_nand_id_table); 2746 2747static int gpmi_nand_probe(struct platform_device *pdev) 2748{ 2749 struct gpmi_nand_data *this; 2750 int ret; 2751 2752 this = devm_kzalloc(&pdev->dev, sizeof(*this), GFP_KERNEL); 2753 if (!this) 2754 return -ENOMEM; 2755 2756 this->devdata = of_device_get_match_data(&pdev->dev); 2757 platform_set_drvdata(pdev, this); 2758 this->pdev = pdev; 2759 this->dev = &pdev->dev; 2760 2761 ret = acquire_resources(this); 2762 if (ret) 2763 goto exit_acquire_resources; 2764 2765 pm_runtime_enable(&pdev->dev); 2766 pm_runtime_set_autosuspend_delay(&pdev->dev, 500); 2767 pm_runtime_use_autosuspend(&pdev->dev); 2768#ifndef CONFIG_PM 2769 ret = gpmi_enable_clk(this); 2770 if (ret) 2771 goto exit_acquire_resources; 2772#endif 2773 2774 ret = gpmi_init(this); 2775 if (ret) 2776 goto exit_nfc_init; 2777 2778 ret = gpmi_nand_init(this); 2779 if (ret) 2780 goto exit_nfc_init; 2781 2782 dev_info(this->dev, "driver registered.\n"); 2783 2784 return 0; 2785 2786exit_nfc_init: 2787 pm_runtime_dont_use_autosuspend(&pdev->dev); 2788 pm_runtime_disable(&pdev->dev); 2789 release_resources(this); 2790exit_acquire_resources: 2791 2792 return ret; 2793} 2794 2795static void gpmi_nand_remove(struct platform_device *pdev) 2796{ 2797 struct gpmi_nand_data *this = platform_get_drvdata(pdev); 2798 struct nand_chip *chip = &this->nand; 2799 int ret; 2800 2801 ret = mtd_device_unregister(nand_to_mtd(chip)); 2802 WARN_ON(ret); 2803 nand_cleanup(chip); 2804 gpmi_free_dma_buffer(this); 2805 release_resources(this); 2806 pm_runtime_dont_use_autosuspend(&pdev->dev); 2807 pm_runtime_disable(&pdev->dev); 2808#ifndef CONFIG_PM 2809 gpmi_disable_clk(this); 2810#endif 2811} 2812 2813static int gpmi_pm_suspend(struct device *dev) 2814{ 2815 int ret; 2816 2817 pinctrl_pm_select_sleep_state(dev); 2818 ret = pm_runtime_force_suspend(dev); 2819 2820 return ret; 2821} 2822 2823static int gpmi_pm_resume(struct device *dev) 2824{ 2825 struct gpmi_nand_data *this = dev_get_drvdata(dev); 2826 int ret; 2827 2828 ret = pm_runtime_force_resume(dev); 2829 if (ret) { 2830 dev_err(this->dev, "Error in resume %d\n", ret); 2831 return ret; 2832 } 2833 2834 pinctrl_pm_select_default_state(dev); 2835 2836 /* re-init the GPMI registers */ 2837 ret = gpmi_init(this); 2838 if (ret) { 2839 dev_err(this->dev, "Error setting GPMI : %d\n", ret); 2840 return ret; 2841 } 2842 2843 /* Set flag to get timing setup restored for next exec_op */ 2844 if (this->hw.clk_rate) 2845 this->hw.must_apply_timings = true; 2846 2847 /* re-init the BCH registers */ 2848 ret = bch_set_geometry(this); 2849 if (ret) { 2850 dev_err(this->dev, "Error setting BCH : %d\n", ret); 2851 return ret; 2852 } 2853 2854 return 0; 2855} 2856 2857static int gpmi_runtime_suspend(struct device *dev) 2858{ 2859 struct gpmi_nand_data *this = dev_get_drvdata(dev); 2860 2861 gpmi_disable_clk(this); 2862 2863 return 0; 2864} 2865 2866static int gpmi_runtime_resume(struct device *dev) 2867{ 2868 struct gpmi_nand_data *this = dev_get_drvdata(dev); 2869 int ret; 2870 2871 ret = gpmi_enable_clk(this); 2872 if (ret) 2873 return ret; 2874 2875 return 0; 2876 2877} 2878 2879static const struct dev_pm_ops gpmi_pm_ops = { 2880 SYSTEM_SLEEP_PM_OPS(gpmi_pm_suspend, gpmi_pm_resume) 2881 RUNTIME_PM_OPS(gpmi_runtime_suspend, gpmi_runtime_resume, NULL) 2882}; 2883 2884static struct platform_driver gpmi_nand_driver = { 2885 .driver = { 2886 .name = "gpmi-nand", 2887 .pm = pm_ptr(&gpmi_pm_ops), 2888 .of_match_table = gpmi_nand_id_table, 2889 }, 2890 .probe = gpmi_nand_probe, 2891 .remove = gpmi_nand_remove, 2892}; 2893module_platform_driver(gpmi_nand_driver); 2894 2895MODULE_AUTHOR("Freescale Semiconductor, Inc."); 2896MODULE_DESCRIPTION("i.MX GPMI NAND Flash Controller Driver"); 2897MODULE_LICENSE("GPL");
~/
Read-only file preview.