FILE
omap2.c
File preview with syntax highlighting.
linux-6.19.14/drivers/mtd/nand/raw/omap2.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright © 2004 Texas Instruments, Jian Zhang <jzhang@ti.com> 4 * Copyright © 2004 Micron Technology Inc. 5 * Copyright © 2004 David Brownell 6 */ 7 8#include <linux/platform_device.h> 9#include <linux/dmaengine.h> 10#include <linux/dma-mapping.h> 11#include <linux/delay.h> 12#include <linux/gpio/consumer.h> 13#include <linux/module.h> 14#include <linux/interrupt.h> 15#include <linux/jiffies.h> 16#include <linux/sched.h> 17#include <linux/mtd/mtd.h> 18#include <linux/mtd/nand-ecc-sw-bch.h> 19#include <linux/mtd/rawnand.h> 20#include <linux/mtd/partitions.h> 21#include <linux/omap-dma.h> 22#include <linux/iopoll.h> 23#include <linux/slab.h> 24#include <linux/of.h> 25#include <linux/of_platform.h> 26 27#include <linux/platform_data/elm.h> 28 29#include <linux/omap-gpmc.h> 30#include <linux/platform_data/mtd-nand-omap2.h> 31 32#define DRIVER_NAME "omap2-nand" 33#define OMAP_NAND_TIMEOUT_MS 5000 34 35#define NAND_Ecc_P1e (1 << 0) 36#define NAND_Ecc_P2e (1 << 1) 37#define NAND_Ecc_P4e (1 << 2) 38#define NAND_Ecc_P8e (1 << 3) 39#define NAND_Ecc_P16e (1 << 4) 40#define NAND_Ecc_P32e (1 << 5) 41#define NAND_Ecc_P64e (1 << 6) 42#define NAND_Ecc_P128e (1 << 7) 43#define NAND_Ecc_P256e (1 << 8) 44#define NAND_Ecc_P512e (1 << 9) 45#define NAND_Ecc_P1024e (1 << 10) 46#define NAND_Ecc_P2048e (1 << 11) 47 48#define NAND_Ecc_P1o (1 << 16) 49#define NAND_Ecc_P2o (1 << 17) 50#define NAND_Ecc_P4o (1 << 18) 51#define NAND_Ecc_P8o (1 << 19) 52#define NAND_Ecc_P16o (1 << 20) 53#define NAND_Ecc_P32o (1 << 21) 54#define NAND_Ecc_P64o (1 << 22) 55#define NAND_Ecc_P128o (1 << 23) 56#define NAND_Ecc_P256o (1 << 24) 57#define NAND_Ecc_P512o (1 << 25) 58#define NAND_Ecc_P1024o (1 << 26) 59#define NAND_Ecc_P2048o (1 << 27) 60 61#define TF(value) (value ? 1 : 0) 62 63#define P2048e(a) (TF(a & NAND_Ecc_P2048e) << 0) 64#define P2048o(a) (TF(a & NAND_Ecc_P2048o) << 1) 65#define P1e(a) (TF(a & NAND_Ecc_P1e) << 2) 66#define P1o(a) (TF(a & NAND_Ecc_P1o) << 3) 67#define P2e(a) (TF(a & NAND_Ecc_P2e) << 4) 68#define P2o(a) (TF(a & NAND_Ecc_P2o) << 5) 69#define P4e(a) (TF(a & NAND_Ecc_P4e) << 6) 70#define P4o(a) (TF(a & NAND_Ecc_P4o) << 7) 71 72#define P8e(a) (TF(a & NAND_Ecc_P8e) << 0) 73#define P8o(a) (TF(a & NAND_Ecc_P8o) << 1) 74#define P16e(a) (TF(a & NAND_Ecc_P16e) << 2) 75#define P16o(a) (TF(a & NAND_Ecc_P16o) << 3) 76#define P32e(a) (TF(a & NAND_Ecc_P32e) << 4) 77#define P32o(a) (TF(a & NAND_Ecc_P32o) << 5) 78#define P64e(a) (TF(a & NAND_Ecc_P64e) << 6) 79#define P64o(a) (TF(a & NAND_Ecc_P64o) << 7) 80 81#define P128e(a) (TF(a & NAND_Ecc_P128e) << 0) 82#define P128o(a) (TF(a & NAND_Ecc_P128o) << 1) 83#define P256e(a) (TF(a & NAND_Ecc_P256e) << 2) 84#define P256o(a) (TF(a & NAND_Ecc_P256o) << 3) 85#define P512e(a) (TF(a & NAND_Ecc_P512e) << 4) 86#define P512o(a) (TF(a & NAND_Ecc_P512o) << 5) 87#define P1024e(a) (TF(a & NAND_Ecc_P1024e) << 6) 88#define P1024o(a) (TF(a & NAND_Ecc_P1024o) << 7) 89 90#define P8e_s(a) (TF(a & NAND_Ecc_P8e) << 0) 91#define P8o_s(a) (TF(a & NAND_Ecc_P8o) << 1) 92#define P16e_s(a) (TF(a & NAND_Ecc_P16e) << 2) 93#define P16o_s(a) (TF(a & NAND_Ecc_P16o) << 3) 94#define P1e_s(a) (TF(a & NAND_Ecc_P1e) << 4) 95#define P1o_s(a) (TF(a & NAND_Ecc_P1o) << 5) 96#define P2e_s(a) (TF(a & NAND_Ecc_P2e) << 6) 97#define P2o_s(a) (TF(a & NAND_Ecc_P2o) << 7) 98 99#define P4e_s(a) (TF(a & NAND_Ecc_P4e) << 0) 100#define P4o_s(a) (TF(a & NAND_Ecc_P4o) << 1) 101 102#define PREFETCH_CONFIG1_CS_SHIFT 24 103#define ECC_CONFIG_CS_SHIFT 1 104#define CS_MASK 0x7 105#define ENABLE_PREFETCH (0x1 << 7) 106#define DMA_MPU_MODE_SHIFT 2 107#define ECCSIZE0_SHIFT 12 108#define ECCSIZE1_SHIFT 22 109#define ECC1RESULTSIZE 0x1 110#define ECCCLEAR 0x100 111#define ECC1 0x1 112#define PREFETCH_FIFOTHRESHOLD_MAX 0x40 113#define PREFETCH_FIFOTHRESHOLD(val) ((val) << 8) 114#define PREFETCH_STATUS_COUNT(val) (val & 0x00003fff) 115#define PREFETCH_STATUS_FIFO_CNT(val) ((val >> 24) & 0x7F) 116#define STATUS_BUFF_EMPTY 0x00000001 117 118#define SECTOR_BYTES 512 119/* 4 bit padding to make byte aligned, 56 = 52 + 4 */ 120#define BCH4_BIT_PAD 4 121 122/* GPMC ecc engine settings for read */ 123#define BCH_WRAPMODE_1 1 /* BCH wrap mode 1 */ 124#define BCH8R_ECC_SIZE0 0x1a /* ecc_size0 = 26 */ 125#define BCH8R_ECC_SIZE1 0x2 /* ecc_size1 = 2 */ 126#define BCH4R_ECC_SIZE0 0xd /* ecc_size0 = 13 */ 127#define BCH4R_ECC_SIZE1 0x3 /* ecc_size1 = 3 */ 128 129/* GPMC ecc engine settings for write */ 130#define BCH_WRAPMODE_6 6 /* BCH wrap mode 6 */ 131#define BCH_ECC_SIZE0 0x0 /* ecc_size0 = 0, no oob protection */ 132#define BCH_ECC_SIZE1 0x20 /* ecc_size1 = 32 */ 133 134#define BBM_LEN 2 135 136static u_char bch16_vector[] = {0xf5, 0x24, 0x1c, 0xd0, 0x61, 0xb3, 0xf1, 0x55, 137 0x2e, 0x2c, 0x86, 0xa3, 0xed, 0x36, 0x1b, 0x78, 138 0x48, 0x76, 0xa9, 0x3b, 0x97, 0xd1, 0x7a, 0x93, 139 0x07, 0x0e}; 140static u_char bch8_vector[] = {0xf3, 0xdb, 0x14, 0x16, 0x8b, 0xd2, 0xbe, 0xcc, 141 0xac, 0x6b, 0xff, 0x99, 0x7b}; 142static u_char bch4_vector[] = {0x00, 0x6b, 0x31, 0xdd, 0x41, 0xbc, 0x10}; 143 144struct omap_nand_info { 145 struct nand_chip nand; 146 struct platform_device *pdev; 147 148 int gpmc_cs; 149 bool dev_ready; 150 enum nand_io xfer_type; 151 enum omap_ecc ecc_opt; 152 struct device_node *elm_of_node; 153 154 unsigned long phys_base; 155 struct completion comp; 156 struct dma_chan *dma; 157 int gpmc_irq_fifo; 158 int gpmc_irq_count; 159 enum { 160 OMAP_NAND_IO_READ = 0, /* read */ 161 OMAP_NAND_IO_WRITE, /* write */ 162 } iomode; 163 u_char *buf; 164 int buf_len; 165 /* Interface to GPMC */ 166 void __iomem *fifo; 167 struct gpmc_nand_regs reg; 168 struct gpmc_nand_ops *ops; 169 bool flash_bbt; 170 /* fields specific for BCHx_HW ECC scheme */ 171 struct device *elm_dev; 172 /* NAND ready gpio */ 173 struct gpio_desc *ready_gpiod; 174 unsigned int neccpg; 175 unsigned int nsteps_per_eccpg; 176 unsigned int eccpg_size; 177 unsigned int eccpg_bytes; 178 void (*data_in)(struct nand_chip *chip, void *buf, 179 unsigned int len, bool force_8bit); 180 void (*data_out)(struct nand_chip *chip, 181 const void *buf, unsigned int len, 182 bool force_8bit); 183}; 184 185static inline struct omap_nand_info *mtd_to_omap(struct mtd_info *mtd) 186{ 187 return container_of(mtd_to_nand(mtd), struct omap_nand_info, nand); 188} 189 190static void omap_nand_data_in(struct nand_chip *chip, void *buf, 191 unsigned int len, bool force_8bit); 192 193static void omap_nand_data_out(struct nand_chip *chip, 194 const void *buf, unsigned int len, 195 bool force_8bit); 196 197/** 198 * omap_prefetch_enable - configures and starts prefetch transfer 199 * @cs: cs (chip select) number 200 * @fifo_th: fifo threshold to be used for read/ write 201 * @dma_mode: dma mode enable (1) or disable (0) 202 * @u32_count: number of bytes to be transferred 203 * @is_write: prefetch read(0) or write post(1) mode 204 * @info: NAND device structure containing platform data 205 */ 206static int omap_prefetch_enable(int cs, int fifo_th, int dma_mode, 207 unsigned int u32_count, int is_write, struct omap_nand_info *info) 208{ 209 u32 val; 210 211 if (fifo_th > PREFETCH_FIFOTHRESHOLD_MAX) 212 return -1; 213 214 if (readl(info->reg.gpmc_prefetch_control)) 215 return -EBUSY; 216 217 /* Set the amount of bytes to be prefetched */ 218 writel(u32_count, info->reg.gpmc_prefetch_config2); 219 220 /* Set dma/mpu mode, the prefetch read / post write and 221 * enable the engine. Set which cs is has requested for. 222 */ 223 val = ((cs << PREFETCH_CONFIG1_CS_SHIFT) | 224 PREFETCH_FIFOTHRESHOLD(fifo_th) | ENABLE_PREFETCH | 225 (dma_mode << DMA_MPU_MODE_SHIFT) | (is_write & 0x1)); 226 writel(val, info->reg.gpmc_prefetch_config1); 227 228 /* Start the prefetch engine */ 229 writel(0x1, info->reg.gpmc_prefetch_control); 230 231 return 0; 232} 233 234/* 235 * omap_prefetch_reset - disables and stops the prefetch engine 236 */ 237static int omap_prefetch_reset(int cs, struct omap_nand_info *info) 238{ 239 u32 config1; 240 241 /* check if the same module/cs is trying to reset */ 242 config1 = readl(info->reg.gpmc_prefetch_config1); 243 if (((config1 >> PREFETCH_CONFIG1_CS_SHIFT) & CS_MASK) != cs) 244 return -EINVAL; 245 246 /* Stop the PFPW engine */ 247 writel(0x0, info->reg.gpmc_prefetch_control); 248 249 /* Reset/disable the PFPW engine */ 250 writel(0x0, info->reg.gpmc_prefetch_config1); 251 252 return 0; 253} 254 255/** 256 * omap_nand_data_in_pref - NAND data in using prefetch engine 257 * @chip: NAND chip 258 * @buf: output buffer where NAND data is placed into 259 * @len: length of transfer 260 * @force_8bit: force 8-bit transfers 261 */ 262static void omap_nand_data_in_pref(struct nand_chip *chip, void *buf, 263 unsigned int len, bool force_8bit) 264{ 265 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 266 uint32_t r_count = 0; 267 int ret = 0; 268 u32 *p = (u32 *)buf; 269 unsigned int pref_len; 270 271 if (force_8bit) { 272 omap_nand_data_in(chip, buf, len, force_8bit); 273 return; 274 } 275 276 /* read 32-bit words using prefetch and remaining bytes normally */ 277 278 /* configure and start prefetch transfer */ 279 pref_len = len - (len & 3); 280 ret = omap_prefetch_enable(info->gpmc_cs, 281 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, pref_len, 0x0, info); 282 if (ret) { 283 /* prefetch engine is busy, use CPU copy method */ 284 omap_nand_data_in(chip, buf, len, false); 285 } else { 286 do { 287 r_count = readl(info->reg.gpmc_prefetch_status); 288 r_count = PREFETCH_STATUS_FIFO_CNT(r_count); 289 r_count = r_count >> 2; 290 ioread32_rep(info->fifo, p, r_count); 291 p += r_count; 292 pref_len -= r_count << 2; 293 } while (pref_len); 294 /* disable and stop the Prefetch engine */ 295 omap_prefetch_reset(info->gpmc_cs, info); 296 /* fetch any remaining bytes */ 297 if (len & 3) 298 omap_nand_data_in(chip, p, len & 3, false); 299 } 300} 301 302/** 303 * omap_nand_data_out_pref - NAND data out using Write Posting engine 304 * @chip: NAND chip 305 * @buf: input buffer that is sent to NAND 306 * @len: length of transfer 307 * @force_8bit: force 8-bit transfers 308 */ 309static void omap_nand_data_out_pref(struct nand_chip *chip, 310 const void *buf, unsigned int len, 311 bool force_8bit) 312{ 313 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 314 uint32_t w_count = 0; 315 int i = 0, ret = 0; 316 u16 *p = (u16 *)buf; 317 unsigned long tim, limit; 318 u32 val; 319 320 if (force_8bit) { 321 omap_nand_data_out(chip, buf, len, force_8bit); 322 return; 323 } 324 325 /* take care of subpage writes */ 326 if (len % 2 != 0) { 327 writeb(*(u8 *)buf, info->fifo); 328 p = (u16 *)(buf + 1); 329 len--; 330 } 331 332 /* configure and start prefetch transfer */ 333 ret = omap_prefetch_enable(info->gpmc_cs, 334 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x1, info); 335 if (ret) { 336 /* write posting engine is busy, use CPU copy method */ 337 omap_nand_data_out(chip, buf, len, false); 338 } else { 339 while (len) { 340 w_count = readl(info->reg.gpmc_prefetch_status); 341 w_count = PREFETCH_STATUS_FIFO_CNT(w_count); 342 w_count = w_count >> 1; 343 for (i = 0; (i < w_count) && len; i++, len -= 2) 344 iowrite16(*p++, info->fifo); 345 } 346 /* wait for data to flushed-out before reset the prefetch */ 347 tim = 0; 348 limit = (loops_per_jiffy * 349 msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 350 do { 351 cpu_relax(); 352 val = readl(info->reg.gpmc_prefetch_status); 353 val = PREFETCH_STATUS_COUNT(val); 354 } while (val && (tim++ < limit)); 355 356 /* disable and stop the PFPW engine */ 357 omap_prefetch_reset(info->gpmc_cs, info); 358 } 359} 360 361/* 362 * omap_nand_dma_callback: callback on the completion of dma transfer 363 * @data: pointer to completion data structure 364 */ 365static void omap_nand_dma_callback(void *data) 366{ 367 complete((struct completion *) data); 368} 369 370/* 371 * omap_nand_dma_transfer: configure and start dma transfer 372 * @chip: nand chip structure 373 * @addr: virtual address in RAM of source/destination 374 * @len: number of data bytes to be transferred 375 * @is_write: flag for read/write operation 376 */ 377static inline int omap_nand_dma_transfer(struct nand_chip *chip, 378 const void *addr, unsigned int len, 379 int is_write) 380{ 381 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 382 struct dma_async_tx_descriptor *tx; 383 enum dma_data_direction dir = is_write ? DMA_TO_DEVICE : 384 DMA_FROM_DEVICE; 385 struct scatterlist sg; 386 unsigned long tim, limit; 387 unsigned n; 388 int ret; 389 u32 val; 390 391 if (!virt_addr_valid(addr)) 392 goto out_copy; 393 394 sg_init_one(&sg, addr, len); 395 n = dma_map_sg(info->dma->device->dev, &sg, 1, dir); 396 if (n == 0) { 397 dev_err(&info->pdev->dev, 398 "Couldn't DMA map a %d byte buffer\n", len); 399 goto out_copy; 400 } 401 402 tx = dmaengine_prep_slave_sg(info->dma, &sg, n, 403 is_write ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM, 404 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 405 if (!tx) 406 goto out_copy_unmap; 407 408 tx->callback = omap_nand_dma_callback; 409 tx->callback_param = &info->comp; 410 dmaengine_submit(tx); 411 412 init_completion(&info->comp); 413 414 /* setup and start DMA using dma_addr */ 415 dma_async_issue_pending(info->dma); 416 417 /* configure and start prefetch transfer */ 418 ret = omap_prefetch_enable(info->gpmc_cs, 419 PREFETCH_FIFOTHRESHOLD_MAX, 0x1, len, is_write, info); 420 if (ret) 421 /* PFPW engine is busy, use cpu copy method */ 422 goto out_copy_unmap; 423 424 wait_for_completion(&info->comp); 425 tim = 0; 426 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 427 428 do { 429 cpu_relax(); 430 val = readl(info->reg.gpmc_prefetch_status); 431 val = PREFETCH_STATUS_COUNT(val); 432 } while (val && (tim++ < limit)); 433 434 /* disable and stop the PFPW engine */ 435 omap_prefetch_reset(info->gpmc_cs, info); 436 437 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir); 438 return 0; 439 440out_copy_unmap: 441 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir); 442out_copy: 443 is_write == 0 ? omap_nand_data_in(chip, (void *)addr, len, false) 444 : omap_nand_data_out(chip, addr, len, false); 445 446 return 0; 447} 448 449/** 450 * omap_nand_data_in_dma_pref - NAND data in using DMA and Prefetch 451 * @chip: NAND chip 452 * @buf: output buffer where NAND data is placed into 453 * @len: length of transfer 454 * @force_8bit: force 8-bit transfers 455 */ 456static void omap_nand_data_in_dma_pref(struct nand_chip *chip, void *buf, 457 unsigned int len, bool force_8bit) 458{ 459 struct mtd_info *mtd = nand_to_mtd(chip); 460 461 if (force_8bit) { 462 omap_nand_data_in(chip, buf, len, force_8bit); 463 return; 464 } 465 466 if (len <= mtd->oobsize) 467 omap_nand_data_in_pref(chip, buf, len, false); 468 else 469 /* start transfer in DMA mode */ 470 omap_nand_dma_transfer(chip, buf, len, 0x0); 471} 472 473/** 474 * omap_nand_data_out_dma_pref - NAND data out using DMA and write posting 475 * @chip: NAND chip 476 * @buf: input buffer that is sent to NAND 477 * @len: length of transfer 478 * @force_8bit: force 8-bit transfers 479 */ 480static void omap_nand_data_out_dma_pref(struct nand_chip *chip, 481 const void *buf, unsigned int len, 482 bool force_8bit) 483{ 484 struct mtd_info *mtd = nand_to_mtd(chip); 485 486 if (force_8bit) { 487 omap_nand_data_out(chip, buf, len, force_8bit); 488 return; 489 } 490 491 if (len <= mtd->oobsize) 492 omap_nand_data_out_pref(chip, buf, len, false); 493 else 494 /* start transfer in DMA mode */ 495 omap_nand_dma_transfer(chip, buf, len, 0x1); 496} 497 498/* 499 * omap_nand_irq - GPMC irq handler 500 * @this_irq: gpmc irq number 501 * @dev: omap_nand_info structure pointer is passed here 502 */ 503static irqreturn_t omap_nand_irq(int this_irq, void *dev) 504{ 505 struct omap_nand_info *info = (struct omap_nand_info *) dev; 506 u32 bytes; 507 508 bytes = readl(info->reg.gpmc_prefetch_status); 509 bytes = PREFETCH_STATUS_FIFO_CNT(bytes); 510 bytes = bytes & 0xFFFC; /* io in multiple of 4 bytes */ 511 if (info->iomode == OMAP_NAND_IO_WRITE) { /* checks for write io */ 512 if (this_irq == info->gpmc_irq_count) 513 goto done; 514 515 if (info->buf_len && (info->buf_len < bytes)) 516 bytes = info->buf_len; 517 else if (!info->buf_len) 518 bytes = 0; 519 iowrite32_rep(info->fifo, (u32 *)info->buf, 520 bytes >> 2); 521 info->buf = info->buf + bytes; 522 info->buf_len -= bytes; 523 524 } else { 525 ioread32_rep(info->fifo, (u32 *)info->buf, 526 bytes >> 2); 527 info->buf = info->buf + bytes; 528 529 if (this_irq == info->gpmc_irq_count) 530 goto done; 531 } 532 533 return IRQ_HANDLED; 534 535done: 536 complete(&info->comp); 537 538 disable_irq_nosync(info->gpmc_irq_fifo); 539 disable_irq_nosync(info->gpmc_irq_count); 540 541 return IRQ_HANDLED; 542} 543 544/* 545 * omap_nand_data_in_irq_pref - NAND data in using Prefetch and IRQ 546 */ 547static void omap_nand_data_in_irq_pref(struct nand_chip *chip, void *buf, 548 unsigned int len, bool force_8bit) 549{ 550 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 551 struct mtd_info *mtd = nand_to_mtd(&info->nand); 552 int ret = 0; 553 554 if (len <= mtd->oobsize || force_8bit) { 555 omap_nand_data_in(chip, buf, len, force_8bit); 556 return; 557 } 558 559 info->iomode = OMAP_NAND_IO_READ; 560 info->buf = buf; 561 init_completion(&info->comp); 562 563 /* configure and start prefetch transfer */ 564 ret = omap_prefetch_enable(info->gpmc_cs, 565 PREFETCH_FIFOTHRESHOLD_MAX/2, 0x0, len, 0x0, info); 566 if (ret) { 567 /* PFPW engine is busy, use cpu copy method */ 568 omap_nand_data_in(chip, buf, len, false); 569 return; 570 } 571 572 info->buf_len = len; 573 574 enable_irq(info->gpmc_irq_count); 575 enable_irq(info->gpmc_irq_fifo); 576 577 /* waiting for read to complete */ 578 wait_for_completion(&info->comp); 579 580 /* disable and stop the PFPW engine */ 581 omap_prefetch_reset(info->gpmc_cs, info); 582 return; 583} 584 585/* 586 * omap_nand_data_out_irq_pref - NAND out using write posting and IRQ 587 */ 588static void omap_nand_data_out_irq_pref(struct nand_chip *chip, 589 const void *buf, unsigned int len, 590 bool force_8bit) 591{ 592 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 593 struct mtd_info *mtd = nand_to_mtd(&info->nand); 594 int ret = 0; 595 unsigned long tim, limit; 596 u32 val; 597 598 if (len <= mtd->oobsize || force_8bit) { 599 omap_nand_data_out(chip, buf, len, force_8bit); 600 return; 601 } 602 603 info->iomode = OMAP_NAND_IO_WRITE; 604 info->buf = (u_char *) buf; 605 init_completion(&info->comp); 606 607 /* configure and start prefetch transfer : size=24 */ 608 ret = omap_prefetch_enable(info->gpmc_cs, 609 (PREFETCH_FIFOTHRESHOLD_MAX * 3) / 8, 0x0, len, 0x1, info); 610 if (ret) { 611 /* PFPW engine is busy, use cpu copy method */ 612 omap_nand_data_out(chip, buf, len, false); 613 return; 614 } 615 616 info->buf_len = len; 617 618 enable_irq(info->gpmc_irq_count); 619 enable_irq(info->gpmc_irq_fifo); 620 621 /* waiting for write to complete */ 622 wait_for_completion(&info->comp); 623 624 /* wait for data to flushed-out before reset the prefetch */ 625 tim = 0; 626 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS)); 627 do { 628 val = readl(info->reg.gpmc_prefetch_status); 629 val = PREFETCH_STATUS_COUNT(val); 630 cpu_relax(); 631 } while (val && (tim++ < limit)); 632 633 /* disable and stop the PFPW engine */ 634 omap_prefetch_reset(info->gpmc_cs, info); 635 return; 636} 637 638/** 639 * gen_true_ecc - This function will generate true ECC value 640 * @ecc_buf: buffer to store ecc code 641 * 642 * This generated true ECC value can be used when correcting 643 * data read from NAND flash memory core 644 */ 645static void gen_true_ecc(u8 *ecc_buf) 646{ 647 u32 tmp = ecc_buf[0] | (ecc_buf[1] << 16) | 648 ((ecc_buf[2] & 0xF0) << 20) | ((ecc_buf[2] & 0x0F) << 8); 649 650 ecc_buf[0] = ~(P64o(tmp) | P64e(tmp) | P32o(tmp) | P32e(tmp) | 651 P16o(tmp) | P16e(tmp) | P8o(tmp) | P8e(tmp)); 652 ecc_buf[1] = ~(P1024o(tmp) | P1024e(tmp) | P512o(tmp) | P512e(tmp) | 653 P256o(tmp) | P256e(tmp) | P128o(tmp) | P128e(tmp)); 654 ecc_buf[2] = ~(P4o(tmp) | P4e(tmp) | P2o(tmp) | P2e(tmp) | P1o(tmp) | 655 P1e(tmp) | P2048o(tmp) | P2048e(tmp)); 656} 657 658/** 659 * omap_compare_ecc - Detect (2 bits) and correct (1 bit) error in data 660 * @ecc_data1: ecc code from nand spare area 661 * @ecc_data2: ecc code from hardware register obtained from hardware ecc 662 * @page_data: page data 663 * 664 * This function compares two ECC's and indicates if there is an error. 665 * If the error can be corrected it will be corrected to the buffer. 666 * If there is no error, %0 is returned. If there is an error but it 667 * was corrected, %1 is returned. Otherwise, %-1 is returned. 668 */ 669static int omap_compare_ecc(u8 *ecc_data1, /* read from NAND memory */ 670 u8 *ecc_data2, /* read from register */ 671 u8 *page_data) 672{ 673 uint i; 674 u8 tmp0_bit[8], tmp1_bit[8], tmp2_bit[8]; 675 u8 comp0_bit[8], comp1_bit[8], comp2_bit[8]; 676 u8 ecc_bit[24]; 677 u8 ecc_sum = 0; 678 u8 find_bit = 0; 679 uint find_byte = 0; 680 int isEccFF; 681 682 isEccFF = ((*(u32 *)ecc_data1 & 0xFFFFFF) == 0xFFFFFF); 683 684 gen_true_ecc(ecc_data1); 685 gen_true_ecc(ecc_data2); 686 687 for (i = 0; i <= 2; i++) { 688 *(ecc_data1 + i) = ~(*(ecc_data1 + i)); 689 *(ecc_data2 + i) = ~(*(ecc_data2 + i)); 690 } 691 692 for (i = 0; i < 8; i++) { 693 tmp0_bit[i] = *ecc_data1 % 2; 694 *ecc_data1 = *ecc_data1 / 2; 695 } 696 697 for (i = 0; i < 8; i++) { 698 tmp1_bit[i] = *(ecc_data1 + 1) % 2; 699 *(ecc_data1 + 1) = *(ecc_data1 + 1) / 2; 700 } 701 702 for (i = 0; i < 8; i++) { 703 tmp2_bit[i] = *(ecc_data1 + 2) % 2; 704 *(ecc_data1 + 2) = *(ecc_data1 + 2) / 2; 705 } 706 707 for (i = 0; i < 8; i++) { 708 comp0_bit[i] = *ecc_data2 % 2; 709 *ecc_data2 = *ecc_data2 / 2; 710 } 711 712 for (i = 0; i < 8; i++) { 713 comp1_bit[i] = *(ecc_data2 + 1) % 2; 714 *(ecc_data2 + 1) = *(ecc_data2 + 1) / 2; 715 } 716 717 for (i = 0; i < 8; i++) { 718 comp2_bit[i] = *(ecc_data2 + 2) % 2; 719 *(ecc_data2 + 2) = *(ecc_data2 + 2) / 2; 720 } 721 722 for (i = 0; i < 6; i++) 723 ecc_bit[i] = tmp2_bit[i + 2] ^ comp2_bit[i + 2]; 724 725 for (i = 0; i < 8; i++) 726 ecc_bit[i + 6] = tmp0_bit[i] ^ comp0_bit[i]; 727 728 for (i = 0; i < 8; i++) 729 ecc_bit[i + 14] = tmp1_bit[i] ^ comp1_bit[i]; 730 731 ecc_bit[22] = tmp2_bit[0] ^ comp2_bit[0]; 732 ecc_bit[23] = tmp2_bit[1] ^ comp2_bit[1]; 733 734 for (i = 0; i < 24; i++) 735 ecc_sum += ecc_bit[i]; 736 737 switch (ecc_sum) { 738 case 0: 739 /* Not reached because this function is not called if 740 * ECC values are equal 741 */ 742 return 0; 743 744 case 1: 745 /* Uncorrectable error */ 746 pr_debug("ECC UNCORRECTED_ERROR 1\n"); 747 return -EBADMSG; 748 749 case 11: 750 /* UN-Correctable error */ 751 pr_debug("ECC UNCORRECTED_ERROR B\n"); 752 return -EBADMSG; 753 754 case 12: 755 /* Correctable error */ 756 find_byte = (ecc_bit[23] << 8) + 757 (ecc_bit[21] << 7) + 758 (ecc_bit[19] << 6) + 759 (ecc_bit[17] << 5) + 760 (ecc_bit[15] << 4) + 761 (ecc_bit[13] << 3) + 762 (ecc_bit[11] << 2) + 763 (ecc_bit[9] << 1) + 764 ecc_bit[7]; 765 766 find_bit = (ecc_bit[5] << 2) + (ecc_bit[3] << 1) + ecc_bit[1]; 767 768 pr_debug("Correcting single bit ECC error at offset: " 769 "%d, bit: %d\n", find_byte, find_bit); 770 771 page_data[find_byte] ^= (1 << find_bit); 772 773 return 1; 774 default: 775 if (isEccFF) { 776 if (ecc_data2[0] == 0 && 777 ecc_data2[1] == 0 && 778 ecc_data2[2] == 0) 779 return 0; 780 } 781 pr_debug("UNCORRECTED_ERROR default\n"); 782 return -EBADMSG; 783 } 784} 785 786/** 787 * omap_correct_data - Compares the ECC read with HW generated ECC 788 * @chip: NAND chip object 789 * @dat: page data 790 * @read_ecc: ecc read from nand flash 791 * @calc_ecc: ecc read from HW ECC registers 792 * 793 * Compares the ecc read from nand spare area with ECC registers values 794 * and if ECC's mismatched, it will call 'omap_compare_ecc' for error 795 * detection and correction. If there are no errors, %0 is returned. If 796 * there were errors and all of the errors were corrected, the number of 797 * corrected errors is returned. If uncorrectable errors exist, %-1 is 798 * returned. 799 */ 800static int omap_correct_data(struct nand_chip *chip, u_char *dat, 801 u_char *read_ecc, u_char *calc_ecc) 802{ 803 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 804 int blockCnt = 0, i = 0, ret = 0; 805 int stat = 0; 806 807 /* Ex NAND_ECC_HW12_2048 */ 808 if (info->nand.ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST && 809 info->nand.ecc.size == 2048) 810 blockCnt = 4; 811 else 812 blockCnt = 1; 813 814 for (i = 0; i < blockCnt; i++) { 815 if (memcmp(read_ecc, calc_ecc, 3) != 0) { 816 ret = omap_compare_ecc(read_ecc, calc_ecc, dat); 817 if (ret < 0) 818 return ret; 819 /* keep track of the number of corrected errors */ 820 stat += ret; 821 } 822 read_ecc += 3; 823 calc_ecc += 3; 824 dat += 512; 825 } 826 return stat; 827} 828 829/** 830 * omap_calculate_ecc - Generate non-inverted ECC bytes. 831 * @chip: NAND chip object 832 * @dat: The pointer to data on which ecc is computed 833 * @ecc_code: The ecc_code buffer 834 * 835 * Using noninverted ECC can be considered ugly since writing a blank 836 * page ie. padding will clear the ECC bytes. This is no problem as long 837 * nobody is trying to write data on the seemingly unused page. Reading 838 * an erased page will produce an ECC mismatch between generated and read 839 * ECC bytes that has to be dealt with separately. 840 */ 841static int omap_calculate_ecc(struct nand_chip *chip, const u_char *dat, 842 u_char *ecc_code) 843{ 844 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 845 u32 val; 846 847 val = readl(info->reg.gpmc_ecc_config); 848 if (((val >> ECC_CONFIG_CS_SHIFT) & CS_MASK) != info->gpmc_cs) 849 return -EINVAL; 850 851 /* read ecc result */ 852 val = readl(info->reg.gpmc_ecc1_result); 853 *ecc_code++ = val; /* P128e, ..., P1e */ 854 *ecc_code++ = val >> 16; /* P128o, ..., P1o */ 855 /* P2048o, P1024o, P512o, P256o, P2048e, P1024e, P512e, P256e */ 856 *ecc_code++ = ((val >> 8) & 0x0f) | ((val >> 20) & 0xf0); 857 858 return 0; 859} 860 861/** 862 * omap_enable_hwecc - This function enables the hardware ecc functionality 863 * @chip: NAND chip object 864 * @mode: Read/Write mode 865 */ 866static void omap_enable_hwecc(struct nand_chip *chip, int mode) 867{ 868 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 869 unsigned int dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0; 870 u32 val; 871 872 /* clear ecc and enable bits */ 873 val = ECCCLEAR | ECC1; 874 writel(val, info->reg.gpmc_ecc_control); 875 876 /* program ecc and result sizes */ 877 val = ((((info->nand.ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) | 878 ECC1RESULTSIZE); 879 writel(val, info->reg.gpmc_ecc_size_config); 880 881 switch (mode) { 882 case NAND_ECC_READ: 883 case NAND_ECC_WRITE: 884 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control); 885 break; 886 case NAND_ECC_READSYN: 887 writel(ECCCLEAR, info->reg.gpmc_ecc_control); 888 break; 889 default: 890 dev_info(&info->pdev->dev, 891 "error: unrecognized Mode[%d]!\n", mode); 892 break; 893 } 894 895 /* (ECC 16 or 8 bit col) | ( CS ) | ECC Enable */ 896 val = (dev_width << 7) | (info->gpmc_cs << 1) | (0x1); 897 writel(val, info->reg.gpmc_ecc_config); 898} 899 900/** 901 * omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation 902 * @chip: NAND chip object 903 * @mode: Read/Write mode 904 * 905 * When using BCH with SW correction (i.e. no ELM), sector size is set 906 * to 512 bytes and we use BCH_WRAPMODE_6 wrapping mode 907 * for both reading and writing with: 908 * eccsize0 = 0 (no additional protected byte in spare area) 909 * eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area) 910 */ 911static void __maybe_unused omap_enable_hwecc_bch(struct nand_chip *chip, 912 int mode) 913{ 914 unsigned int bch_type; 915 unsigned int dev_width, nsectors; 916 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 917 enum omap_ecc ecc_opt = info->ecc_opt; 918 u32 val, wr_mode; 919 unsigned int ecc_size1, ecc_size0; 920 921 /* GPMC configurations for calculating ECC */ 922 switch (ecc_opt) { 923 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 924 bch_type = 0; 925 nsectors = 1; 926 wr_mode = BCH_WRAPMODE_6; 927 ecc_size0 = BCH_ECC_SIZE0; 928 ecc_size1 = BCH_ECC_SIZE1; 929 break; 930 case OMAP_ECC_BCH4_CODE_HW: 931 bch_type = 0; 932 nsectors = chip->ecc.steps; 933 if (mode == NAND_ECC_READ) { 934 wr_mode = BCH_WRAPMODE_1; 935 ecc_size0 = BCH4R_ECC_SIZE0; 936 ecc_size1 = BCH4R_ECC_SIZE1; 937 } else { 938 wr_mode = BCH_WRAPMODE_6; 939 ecc_size0 = BCH_ECC_SIZE0; 940 ecc_size1 = BCH_ECC_SIZE1; 941 } 942 break; 943 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 944 bch_type = 1; 945 nsectors = 1; 946 wr_mode = BCH_WRAPMODE_6; 947 ecc_size0 = BCH_ECC_SIZE0; 948 ecc_size1 = BCH_ECC_SIZE1; 949 break; 950 case OMAP_ECC_BCH8_CODE_HW: 951 bch_type = 1; 952 nsectors = chip->ecc.steps; 953 if (mode == NAND_ECC_READ) { 954 wr_mode = BCH_WRAPMODE_1; 955 ecc_size0 = BCH8R_ECC_SIZE0; 956 ecc_size1 = BCH8R_ECC_SIZE1; 957 } else { 958 wr_mode = BCH_WRAPMODE_6; 959 ecc_size0 = BCH_ECC_SIZE0; 960 ecc_size1 = BCH_ECC_SIZE1; 961 } 962 break; 963 case OMAP_ECC_BCH16_CODE_HW: 964 bch_type = 0x2; 965 nsectors = chip->ecc.steps; 966 if (mode == NAND_ECC_READ) { 967 wr_mode = 0x01; 968 ecc_size0 = 52; /* ECC bits in nibbles per sector */ 969 ecc_size1 = 0; /* non-ECC bits in nibbles per sector */ 970 } else { 971 wr_mode = 0x01; 972 ecc_size0 = 0; /* extra bits in nibbles per sector */ 973 ecc_size1 = 52; /* OOB bits in nibbles per sector */ 974 } 975 break; 976 default: 977 return; 978 } 979 980 writel(ECC1, info->reg.gpmc_ecc_control); 981 982 /* Configure ecc size for BCH */ 983 val = (ecc_size1 << ECCSIZE1_SHIFT) | (ecc_size0 << ECCSIZE0_SHIFT); 984 writel(val, info->reg.gpmc_ecc_size_config); 985 986 dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0; 987 988 /* BCH configuration */ 989 val = ((1 << 16) | /* enable BCH */ 990 (bch_type << 12) | /* BCH4/BCH8/BCH16 */ 991 (wr_mode << 8) | /* wrap mode */ 992 (dev_width << 7) | /* bus width */ 993 (((nsectors-1) & 0x7) << 4) | /* number of sectors */ 994 (info->gpmc_cs << 1) | /* ECC CS */ 995 (0x1)); /* enable ECC */ 996 997 writel(val, info->reg.gpmc_ecc_config); 998 999 /* Clear ecc and enable bits */ 1000 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control); 1001} 1002 1003static u8 bch4_polynomial[] = {0x28, 0x13, 0xcc, 0x39, 0x96, 0xac, 0x7f}; 1004static u8 bch8_polynomial[] = {0xef, 0x51, 0x2e, 0x09, 0xed, 0x93, 0x9a, 0xc2, 1005 0x97, 0x79, 0xe5, 0x24, 0xb5}; 1006 1007/** 1008 * _omap_calculate_ecc_bch - Generate ECC bytes for one sector 1009 * @mtd: MTD device structure 1010 * @dat: The pointer to data on which ecc is computed 1011 * @ecc_calc: The ecc_code buffer 1012 * @i: The sector number (for a multi sector page) 1013 * 1014 * Support calculating of BCH4/8/16 ECC vectors for one sector 1015 * within a page. Sector number is in @i. 1016 */ 1017static int _omap_calculate_ecc_bch(struct mtd_info *mtd, 1018 const u_char *dat, u_char *ecc_calc, int i) 1019{ 1020 struct omap_nand_info *info = mtd_to_omap(mtd); 1021 int eccbytes = info->nand.ecc.bytes; 1022 struct gpmc_nand_regs *gpmc_regs = &info->reg; 1023 u8 *ecc_code; 1024 unsigned long bch_val1, bch_val2, bch_val3, bch_val4; 1025 u32 val; 1026 int j; 1027 1028 ecc_code = ecc_calc; 1029 switch (info->ecc_opt) { 1030 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1031 case OMAP_ECC_BCH8_CODE_HW: 1032 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]); 1033 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]); 1034 bch_val3 = readl(gpmc_regs->gpmc_bch_result2[i]); 1035 bch_val4 = readl(gpmc_regs->gpmc_bch_result3[i]); 1036 *ecc_code++ = (bch_val4 & 0xFF); 1037 *ecc_code++ = ((bch_val3 >> 24) & 0xFF); 1038 *ecc_code++ = ((bch_val3 >> 16) & 0xFF); 1039 *ecc_code++ = ((bch_val3 >> 8) & 0xFF); 1040 *ecc_code++ = (bch_val3 & 0xFF); 1041 *ecc_code++ = ((bch_val2 >> 24) & 0xFF); 1042 *ecc_code++ = ((bch_val2 >> 16) & 0xFF); 1043 *ecc_code++ = ((bch_val2 >> 8) & 0xFF); 1044 *ecc_code++ = (bch_val2 & 0xFF); 1045 *ecc_code++ = ((bch_val1 >> 24) & 0xFF); 1046 *ecc_code++ = ((bch_val1 >> 16) & 0xFF); 1047 *ecc_code++ = ((bch_val1 >> 8) & 0xFF); 1048 *ecc_code++ = (bch_val1 & 0xFF); 1049 break; 1050 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1051 case OMAP_ECC_BCH4_CODE_HW: 1052 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]); 1053 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]); 1054 *ecc_code++ = ((bch_val2 >> 12) & 0xFF); 1055 *ecc_code++ = ((bch_val2 >> 4) & 0xFF); 1056 *ecc_code++ = ((bch_val2 & 0xF) << 4) | 1057 ((bch_val1 >> 28) & 0xF); 1058 *ecc_code++ = ((bch_val1 >> 20) & 0xFF); 1059 *ecc_code++ = ((bch_val1 >> 12) & 0xFF); 1060 *ecc_code++ = ((bch_val1 >> 4) & 0xFF); 1061 *ecc_code++ = ((bch_val1 & 0xF) << 4); 1062 break; 1063 case OMAP_ECC_BCH16_CODE_HW: 1064 val = readl(gpmc_regs->gpmc_bch_result6[i]); 1065 ecc_code[0] = ((val >> 8) & 0xFF); 1066 ecc_code[1] = ((val >> 0) & 0xFF); 1067 val = readl(gpmc_regs->gpmc_bch_result5[i]); 1068 ecc_code[2] = ((val >> 24) & 0xFF); 1069 ecc_code[3] = ((val >> 16) & 0xFF); 1070 ecc_code[4] = ((val >> 8) & 0xFF); 1071 ecc_code[5] = ((val >> 0) & 0xFF); 1072 val = readl(gpmc_regs->gpmc_bch_result4[i]); 1073 ecc_code[6] = ((val >> 24) & 0xFF); 1074 ecc_code[7] = ((val >> 16) & 0xFF); 1075 ecc_code[8] = ((val >> 8) & 0xFF); 1076 ecc_code[9] = ((val >> 0) & 0xFF); 1077 val = readl(gpmc_regs->gpmc_bch_result3[i]); 1078 ecc_code[10] = ((val >> 24) & 0xFF); 1079 ecc_code[11] = ((val >> 16) & 0xFF); 1080 ecc_code[12] = ((val >> 8) & 0xFF); 1081 ecc_code[13] = ((val >> 0) & 0xFF); 1082 val = readl(gpmc_regs->gpmc_bch_result2[i]); 1083 ecc_code[14] = ((val >> 24) & 0xFF); 1084 ecc_code[15] = ((val >> 16) & 0xFF); 1085 ecc_code[16] = ((val >> 8) & 0xFF); 1086 ecc_code[17] = ((val >> 0) & 0xFF); 1087 val = readl(gpmc_regs->gpmc_bch_result1[i]); 1088 ecc_code[18] = ((val >> 24) & 0xFF); 1089 ecc_code[19] = ((val >> 16) & 0xFF); 1090 ecc_code[20] = ((val >> 8) & 0xFF); 1091 ecc_code[21] = ((val >> 0) & 0xFF); 1092 val = readl(gpmc_regs->gpmc_bch_result0[i]); 1093 ecc_code[22] = ((val >> 24) & 0xFF); 1094 ecc_code[23] = ((val >> 16) & 0xFF); 1095 ecc_code[24] = ((val >> 8) & 0xFF); 1096 ecc_code[25] = ((val >> 0) & 0xFF); 1097 break; 1098 default: 1099 return -EINVAL; 1100 } 1101 1102 /* ECC scheme specific syndrome customizations */ 1103 switch (info->ecc_opt) { 1104 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1105 /* Add constant polynomial to remainder, so that 1106 * ECC of blank pages results in 0x0 on reading back 1107 */ 1108 for (j = 0; j < eccbytes; j++) 1109 ecc_calc[j] ^= bch4_polynomial[j]; 1110 break; 1111 case OMAP_ECC_BCH4_CODE_HW: 1112 /* Set 8th ECC byte as 0x0 for ROM compatibility */ 1113 ecc_calc[eccbytes - 1] = 0x0; 1114 break; 1115 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1116 /* Add constant polynomial to remainder, so that 1117 * ECC of blank pages results in 0x0 on reading back 1118 */ 1119 for (j = 0; j < eccbytes; j++) 1120 ecc_calc[j] ^= bch8_polynomial[j]; 1121 break; 1122 case OMAP_ECC_BCH8_CODE_HW: 1123 /* Set 14th ECC byte as 0x0 for ROM compatibility */ 1124 ecc_calc[eccbytes - 1] = 0x0; 1125 break; 1126 case OMAP_ECC_BCH16_CODE_HW: 1127 break; 1128 default: 1129 return -EINVAL; 1130 } 1131 1132 return 0; 1133} 1134 1135/** 1136 * omap_calculate_ecc_bch_sw - ECC generator for sector for SW based correction 1137 * @chip: NAND chip object 1138 * @dat: The pointer to data on which ecc is computed 1139 * @ecc_calc: Buffer storing the calculated ECC bytes 1140 * 1141 * Support calculating of BCH4/8/16 ECC vectors for one sector. This is used 1142 * when SW based correction is required as ECC is required for one sector 1143 * at a time. 1144 */ 1145static int omap_calculate_ecc_bch_sw(struct nand_chip *chip, 1146 const u_char *dat, u_char *ecc_calc) 1147{ 1148 return _omap_calculate_ecc_bch(nand_to_mtd(chip), dat, ecc_calc, 0); 1149} 1150 1151/** 1152 * omap_calculate_ecc_bch_multi - Generate ECC for multiple sectors 1153 * @mtd: MTD device structure 1154 * @dat: The pointer to data on which ecc is computed 1155 * @ecc_calc: Buffer storing the calculated ECC bytes 1156 * 1157 * Support calculating of BCH4/8/16 ecc vectors for the entire page in one go. 1158 */ 1159static int omap_calculate_ecc_bch_multi(struct mtd_info *mtd, 1160 const u_char *dat, u_char *ecc_calc) 1161{ 1162 struct omap_nand_info *info = mtd_to_omap(mtd); 1163 int eccbytes = info->nand.ecc.bytes; 1164 unsigned long nsectors; 1165 int i, ret; 1166 1167 nsectors = ((readl(info->reg.gpmc_ecc_config) >> 4) & 0x7) + 1; 1168 for (i = 0; i < nsectors; i++) { 1169 ret = _omap_calculate_ecc_bch(mtd, dat, ecc_calc, i); 1170 if (ret) 1171 return ret; 1172 1173 ecc_calc += eccbytes; 1174 } 1175 1176 return 0; 1177} 1178 1179/** 1180 * erased_sector_bitflips - count bit flips 1181 * @data: data sector buffer 1182 * @oob: oob buffer 1183 * @info: omap_nand_info 1184 * 1185 * Check the bit flips in erased page falls below correctable level. 1186 * If falls below, report the page as erased with correctable bit 1187 * flip, else report as uncorrectable page. 1188 */ 1189static int erased_sector_bitflips(u_char *data, u_char *oob, 1190 struct omap_nand_info *info) 1191{ 1192 int flip_bits = 0, i; 1193 1194 for (i = 0; i < info->nand.ecc.size; i++) { 1195 flip_bits += hweight8(~data[i]); 1196 if (flip_bits > info->nand.ecc.strength) 1197 return 0; 1198 } 1199 1200 for (i = 0; i < info->nand.ecc.bytes - 1; i++) { 1201 flip_bits += hweight8(~oob[i]); 1202 if (flip_bits > info->nand.ecc.strength) 1203 return 0; 1204 } 1205 1206 /* 1207 * Bit flips falls in correctable level. 1208 * Fill data area with 0xFF 1209 */ 1210 if (flip_bits) { 1211 memset(data, 0xFF, info->nand.ecc.size); 1212 memset(oob, 0xFF, info->nand.ecc.bytes); 1213 } 1214 1215 return flip_bits; 1216} 1217 1218/** 1219 * omap_elm_correct_data - corrects page data area in case error reported 1220 * @chip: NAND chip object 1221 * @data: page data 1222 * @read_ecc: ecc read from nand flash 1223 * @calc_ecc: ecc read from HW ECC registers 1224 * 1225 * Calculated ecc vector reported as zero in case of non-error pages. 1226 * In case of non-zero ecc vector, first filter out erased-pages, and 1227 * then process data via ELM to detect bit-flips. 1228 */ 1229static int omap_elm_correct_data(struct nand_chip *chip, u_char *data, 1230 u_char *read_ecc, u_char *calc_ecc) 1231{ 1232 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 1233 struct nand_ecc_ctrl *ecc = &info->nand.ecc; 1234 int eccsteps = info->nsteps_per_eccpg; 1235 int i , j, stat = 0; 1236 int eccflag, actual_eccbytes; 1237 struct elm_errorvec err_vec[ERROR_VECTOR_MAX]; 1238 u_char *ecc_vec = calc_ecc; 1239 u_char *spare_ecc = read_ecc; 1240 u_char *erased_ecc_vec; 1241 u_char *buf; 1242 int bitflip_count; 1243 bool is_error_reported = false; 1244 u32 bit_pos, byte_pos, error_max, pos; 1245 int err; 1246 1247 switch (info->ecc_opt) { 1248 case OMAP_ECC_BCH4_CODE_HW: 1249 /* omit 7th ECC byte reserved for ROM code compatibility */ 1250 actual_eccbytes = ecc->bytes - 1; 1251 erased_ecc_vec = bch4_vector; 1252 break; 1253 case OMAP_ECC_BCH8_CODE_HW: 1254 /* omit 14th ECC byte reserved for ROM code compatibility */ 1255 actual_eccbytes = ecc->bytes - 1; 1256 erased_ecc_vec = bch8_vector; 1257 break; 1258 case OMAP_ECC_BCH16_CODE_HW: 1259 actual_eccbytes = ecc->bytes; 1260 erased_ecc_vec = bch16_vector; 1261 break; 1262 default: 1263 dev_err(&info->pdev->dev, "invalid driver configuration\n"); 1264 return -EINVAL; 1265 } 1266 1267 /* Initialize elm error vector to zero */ 1268 memset(err_vec, 0, sizeof(err_vec)); 1269 1270 for (i = 0; i < eccsteps ; i++) { 1271 eccflag = 0; /* initialize eccflag */ 1272 1273 /* 1274 * Check any error reported, 1275 * In case of error, non zero ecc reported. 1276 */ 1277 for (j = 0; j < actual_eccbytes; j++) { 1278 if (calc_ecc[j] != 0) { 1279 eccflag = 1; /* non zero ecc, error present */ 1280 break; 1281 } 1282 } 1283 1284 if (eccflag == 1) { 1285 if (memcmp(calc_ecc, erased_ecc_vec, 1286 actual_eccbytes) == 0) { 1287 /* 1288 * calc_ecc[] matches pattern for ECC(all 0xff) 1289 * so this is definitely an erased-page 1290 */ 1291 } else { 1292 buf = &data[info->nand.ecc.size * i]; 1293 /* 1294 * count number of 0-bits in read_buf. 1295 * This check can be removed once a similar 1296 * check is introduced in generic NAND driver 1297 */ 1298 bitflip_count = erased_sector_bitflips( 1299 buf, read_ecc, info); 1300 if (bitflip_count) { 1301 /* 1302 * number of 0-bits within ECC limits 1303 * So this may be an erased-page 1304 */ 1305 stat += bitflip_count; 1306 } else { 1307 /* 1308 * Too many 0-bits. It may be a 1309 * - programmed-page, OR 1310 * - erased-page with many bit-flips 1311 * So this page requires check by ELM 1312 */ 1313 err_vec[i].error_reported = true; 1314 is_error_reported = true; 1315 } 1316 } 1317 } 1318 1319 /* Update the ecc vector */ 1320 calc_ecc += ecc->bytes; 1321 read_ecc += ecc->bytes; 1322 } 1323 1324 /* Check if any error reported */ 1325 if (!is_error_reported) 1326 return stat; 1327 1328 /* Decode BCH error using ELM module */ 1329 elm_decode_bch_error_page(info->elm_dev, ecc_vec, err_vec); 1330 1331 err = 0; 1332 for (i = 0; i < eccsteps; i++) { 1333 if (err_vec[i].error_uncorrectable) { 1334 dev_err(&info->pdev->dev, 1335 "uncorrectable bit-flips found\n"); 1336 err = -EBADMSG; 1337 } else if (err_vec[i].error_reported) { 1338 for (j = 0; j < err_vec[i].error_count; j++) { 1339 switch (info->ecc_opt) { 1340 case OMAP_ECC_BCH4_CODE_HW: 1341 /* Add 4 bits to take care of padding */ 1342 pos = err_vec[i].error_loc[j] + 1343 BCH4_BIT_PAD; 1344 break; 1345 case OMAP_ECC_BCH8_CODE_HW: 1346 case OMAP_ECC_BCH16_CODE_HW: 1347 pos = err_vec[i].error_loc[j]; 1348 break; 1349 default: 1350 return -EINVAL; 1351 } 1352 error_max = (ecc->size + actual_eccbytes) * 8; 1353 /* Calculate bit position of error */ 1354 bit_pos = pos % 8; 1355 1356 /* Calculate byte position of error */ 1357 byte_pos = (error_max - pos - 1) / 8; 1358 1359 if (pos < error_max) { 1360 if (byte_pos < 512) { 1361 pr_debug("bitflip@dat[%d]=%x\n", 1362 byte_pos, data[byte_pos]); 1363 data[byte_pos] ^= 1 << bit_pos; 1364 } else { 1365 pr_debug("bitflip@oob[%d]=%x\n", 1366 (byte_pos - 512), 1367 spare_ecc[byte_pos - 512]); 1368 spare_ecc[byte_pos - 512] ^= 1369 1 << bit_pos; 1370 } 1371 } else { 1372 dev_err(&info->pdev->dev, 1373 "invalid bit-flip @ %d:%d\n", 1374 byte_pos, bit_pos); 1375 err = -EBADMSG; 1376 } 1377 } 1378 } 1379 1380 /* Update number of correctable errors */ 1381 stat = max_t(unsigned int, stat, err_vec[i].error_count); 1382 1383 /* Update page data with sector size */ 1384 data += ecc->size; 1385 spare_ecc += ecc->bytes; 1386 } 1387 1388 return (err) ? err : stat; 1389} 1390 1391/** 1392 * omap_write_page_bch - BCH ecc based write page function for entire page 1393 * @chip: nand chip info structure 1394 * @buf: data buffer 1395 * @oob_required: must write chip->oob_poi to OOB 1396 * @page: page 1397 * 1398 * Custom write page method evolved to support multi sector writing in one shot 1399 */ 1400static int omap_write_page_bch(struct nand_chip *chip, const uint8_t *buf, 1401 int oob_required, int page) 1402{ 1403 struct mtd_info *mtd = nand_to_mtd(chip); 1404 struct omap_nand_info *info = mtd_to_omap(mtd); 1405 uint8_t *ecc_calc = chip->ecc.calc_buf; 1406 unsigned int eccpg; 1407 int ret; 1408 1409 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1410 if (ret) 1411 return ret; 1412 1413 for (eccpg = 0; eccpg < info->neccpg; eccpg++) { 1414 /* Enable GPMC ecc engine */ 1415 chip->ecc.hwctl(chip, NAND_ECC_WRITE); 1416 1417 /* Write data */ 1418 info->data_out(chip, buf + (eccpg * info->eccpg_size), 1419 info->eccpg_size, false); 1420 1421 /* Update ecc vector from GPMC result registers */ 1422 ret = omap_calculate_ecc_bch_multi(mtd, 1423 buf + (eccpg * info->eccpg_size), 1424 ecc_calc); 1425 if (ret) 1426 return ret; 1427 1428 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, 1429 chip->oob_poi, 1430 eccpg * info->eccpg_bytes, 1431 info->eccpg_bytes); 1432 if (ret) 1433 return ret; 1434 } 1435 1436 /* Write ecc vector to OOB area */ 1437 info->data_out(chip, chip->oob_poi, mtd->oobsize, false); 1438 1439 return nand_prog_page_end_op(chip); 1440} 1441 1442/** 1443 * omap_write_subpage_bch - BCH hardware ECC based subpage write 1444 * @chip: nand chip info structure 1445 * @offset: column address of subpage within the page 1446 * @data_len: data length 1447 * @buf: data buffer 1448 * @oob_required: must write chip->oob_poi to OOB 1449 * @page: page number to write 1450 * 1451 * OMAP optimized subpage write method. 1452 */ 1453static int omap_write_subpage_bch(struct nand_chip *chip, u32 offset, 1454 u32 data_len, const u8 *buf, 1455 int oob_required, int page) 1456{ 1457 struct mtd_info *mtd = nand_to_mtd(chip); 1458 struct omap_nand_info *info = mtd_to_omap(mtd); 1459 u8 *ecc_calc = chip->ecc.calc_buf; 1460 int ecc_size = chip->ecc.size; 1461 int ecc_bytes = chip->ecc.bytes; 1462 u32 start_step = offset / ecc_size; 1463 u32 end_step = (offset + data_len - 1) / ecc_size; 1464 unsigned int eccpg; 1465 int step, ret = 0; 1466 1467 /* 1468 * Write entire page at one go as it would be optimal 1469 * as ECC is calculated by hardware. 1470 * ECC is calculated for all subpages but we choose 1471 * only what we want. 1472 */ 1473 ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0); 1474 if (ret) 1475 return ret; 1476 1477 for (eccpg = 0; eccpg < info->neccpg; eccpg++) { 1478 /* Enable GPMC ECC engine */ 1479 chip->ecc.hwctl(chip, NAND_ECC_WRITE); 1480 1481 /* Write data */ 1482 info->data_out(chip, buf + (eccpg * info->eccpg_size), 1483 info->eccpg_size, false); 1484 1485 for (step = 0; step < info->nsteps_per_eccpg; step++) { 1486 unsigned int base_step = eccpg * info->nsteps_per_eccpg; 1487 const u8 *bufoffs = buf + (eccpg * info->eccpg_size); 1488 1489 /* Mask ECC of un-touched subpages with 0xFFs */ 1490 if ((step + base_step) < start_step || 1491 (step + base_step) > end_step) 1492 memset(ecc_calc + (step * ecc_bytes), 0xff, 1493 ecc_bytes); 1494 else 1495 ret = _omap_calculate_ecc_bch(mtd, 1496 bufoffs + (step * ecc_size), 1497 ecc_calc + (step * ecc_bytes), 1498 step); 1499 1500 if (ret) 1501 return ret; 1502 } 1503 1504 /* 1505 * Copy the calculated ECC for the whole page including the 1506 * masked values (0xFF) corresponding to unwritten subpages. 1507 */ 1508 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 1509 eccpg * info->eccpg_bytes, 1510 info->eccpg_bytes); 1511 if (ret) 1512 return ret; 1513 } 1514 1515 /* write OOB buffer to NAND device */ 1516 info->data_out(chip, chip->oob_poi, mtd->oobsize, false); 1517 1518 return nand_prog_page_end_op(chip); 1519} 1520 1521/** 1522 * omap_read_page_bch - BCH ecc based page read function for entire page 1523 * @chip: nand chip info structure 1524 * @buf: buffer to store read data 1525 * @oob_required: caller requires OOB data read to chip->oob_poi 1526 * @page: page number to read 1527 * 1528 * For BCH ecc scheme, GPMC used for syndrome calculation and ELM module 1529 * used for error correction. 1530 * Custom method evolved to support ELM error correction & multi sector 1531 * reading. On reading page data area is read along with OOB data with 1532 * ecc engine enabled. ecc vector updated after read of OOB data. 1533 * For non error pages ecc vector reported as zero. 1534 */ 1535static int omap_read_page_bch(struct nand_chip *chip, uint8_t *buf, 1536 int oob_required, int page) 1537{ 1538 struct mtd_info *mtd = nand_to_mtd(chip); 1539 struct omap_nand_info *info = mtd_to_omap(mtd); 1540 uint8_t *ecc_calc = chip->ecc.calc_buf; 1541 uint8_t *ecc_code = chip->ecc.code_buf; 1542 unsigned int max_bitflips = 0, eccpg; 1543 int stat, ret; 1544 1545 ret = nand_read_page_op(chip, page, 0, NULL, 0); 1546 if (ret) 1547 return ret; 1548 1549 for (eccpg = 0; eccpg < info->neccpg; eccpg++) { 1550 /* Enable GPMC ecc engine */ 1551 chip->ecc.hwctl(chip, NAND_ECC_READ); 1552 1553 /* Read data */ 1554 ret = nand_change_read_column_op(chip, eccpg * info->eccpg_size, 1555 buf + (eccpg * info->eccpg_size), 1556 info->eccpg_size, false); 1557 if (ret) 1558 return ret; 1559 1560 /* Read oob bytes */ 1561 ret = nand_change_read_column_op(chip, 1562 mtd->writesize + BBM_LEN + 1563 (eccpg * info->eccpg_bytes), 1564 chip->oob_poi + BBM_LEN + 1565 (eccpg * info->eccpg_bytes), 1566 info->eccpg_bytes, false); 1567 if (ret) 1568 return ret; 1569 1570 /* Calculate ecc bytes */ 1571 ret = omap_calculate_ecc_bch_multi(mtd, 1572 buf + (eccpg * info->eccpg_size), 1573 ecc_calc); 1574 if (ret) 1575 return ret; 1576 1577 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, 1578 chip->oob_poi, 1579 eccpg * info->eccpg_bytes, 1580 info->eccpg_bytes); 1581 if (ret) 1582 return ret; 1583 1584 stat = chip->ecc.correct(chip, 1585 buf + (eccpg * info->eccpg_size), 1586 ecc_code, ecc_calc); 1587 if (stat < 0) { 1588 mtd->ecc_stats.failed++; 1589 } else { 1590 mtd->ecc_stats.corrected += stat; 1591 max_bitflips = max_t(unsigned int, max_bitflips, stat); 1592 } 1593 } 1594 1595 return max_bitflips; 1596} 1597 1598/** 1599 * is_elm_present - checks for presence of ELM module by scanning DT nodes 1600 * @info: NAND device structure containing platform data 1601 * @elm_node: ELM's DT node 1602 */ 1603static bool is_elm_present(struct omap_nand_info *info, 1604 struct device_node *elm_node) 1605{ 1606 struct platform_device *pdev; 1607 1608 /* check whether elm-id is passed via DT */ 1609 if (!elm_node) { 1610 dev_err(&info->pdev->dev, "ELM devicetree node not found\n"); 1611 return false; 1612 } 1613 pdev = of_find_device_by_node(elm_node); 1614 /* check whether ELM device is registered */ 1615 if (!pdev) { 1616 dev_err(&info->pdev->dev, "ELM device not found\n"); 1617 return false; 1618 } 1619 /* ELM module available, now configure it */ 1620 info->elm_dev = &pdev->dev; 1621 return true; 1622} 1623 1624static bool omap2_nand_ecc_check(struct omap_nand_info *info) 1625{ 1626 bool ecc_needs_bch, ecc_needs_omap_bch, ecc_needs_elm; 1627 1628 switch (info->ecc_opt) { 1629 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1630 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 1631 ecc_needs_omap_bch = false; 1632 ecc_needs_bch = true; 1633 ecc_needs_elm = false; 1634 break; 1635 case OMAP_ECC_BCH4_CODE_HW: 1636 case OMAP_ECC_BCH8_CODE_HW: 1637 case OMAP_ECC_BCH16_CODE_HW: 1638 ecc_needs_omap_bch = true; 1639 ecc_needs_bch = false; 1640 ecc_needs_elm = true; 1641 break; 1642 default: 1643 ecc_needs_omap_bch = false; 1644 ecc_needs_bch = false; 1645 ecc_needs_elm = false; 1646 break; 1647 } 1648 1649 if (ecc_needs_bch && !IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_BCH)) { 1650 dev_err(&info->pdev->dev, 1651 "CONFIG_MTD_NAND_ECC_SW_BCH not enabled\n"); 1652 return false; 1653 } 1654 if (ecc_needs_omap_bch && !IS_ENABLED(CONFIG_MTD_NAND_OMAP_BCH)) { 1655 dev_err(&info->pdev->dev, 1656 "CONFIG_MTD_NAND_OMAP_BCH not enabled\n"); 1657 return false; 1658 } 1659 if (ecc_needs_elm && !is_elm_present(info, info->elm_of_node)) { 1660 dev_err(&info->pdev->dev, "ELM not available\n"); 1661 return false; 1662 } 1663 1664 return true; 1665} 1666 1667static const char * const nand_xfer_types[] = { 1668 [NAND_OMAP_PREFETCH_POLLED] = "prefetch-polled", 1669 [NAND_OMAP_POLLED] = "polled", 1670 [NAND_OMAP_PREFETCH_DMA] = "prefetch-dma", 1671 [NAND_OMAP_PREFETCH_IRQ] = "prefetch-irq", 1672}; 1673 1674static int omap_get_dt_info(struct device *dev, struct omap_nand_info *info) 1675{ 1676 struct device_node *child = dev->of_node; 1677 int i; 1678 const char *s; 1679 u32 cs; 1680 1681 if (of_property_read_u32(child, "reg", &cs) < 0) { 1682 dev_err(dev, "reg not found in DT\n"); 1683 return -EINVAL; 1684 } 1685 1686 info->gpmc_cs = cs; 1687 1688 /* detect availability of ELM module. Won't be present pre-OMAP4 */ 1689 info->elm_of_node = of_parse_phandle(child, "ti,elm-id", 0); 1690 if (!info->elm_of_node) { 1691 info->elm_of_node = of_parse_phandle(child, "elm_id", 0); 1692 if (!info->elm_of_node) 1693 dev_dbg(dev, "ti,elm-id not in DT\n"); 1694 } 1695 1696 /* select ecc-scheme for NAND */ 1697 if (of_property_read_string(child, "ti,nand-ecc-opt", &s)) { 1698 dev_err(dev, "ti,nand-ecc-opt not found\n"); 1699 return -EINVAL; 1700 } 1701 1702 if (!strcmp(s, "sw")) { 1703 info->ecc_opt = OMAP_ECC_HAM1_CODE_SW; 1704 } else if (!strcmp(s, "ham1") || 1705 !strcmp(s, "hw") || !strcmp(s, "hw-romcode")) { 1706 info->ecc_opt = OMAP_ECC_HAM1_CODE_HW; 1707 } else if (!strcmp(s, "bch4")) { 1708 if (info->elm_of_node) 1709 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW; 1710 else 1711 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW_DETECTION_SW; 1712 } else if (!strcmp(s, "bch8")) { 1713 if (info->elm_of_node) 1714 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW; 1715 else 1716 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW_DETECTION_SW; 1717 } else if (!strcmp(s, "bch16")) { 1718 info->ecc_opt = OMAP_ECC_BCH16_CODE_HW; 1719 } else { 1720 dev_err(dev, "unrecognized value for ti,nand-ecc-opt\n"); 1721 return -EINVAL; 1722 } 1723 1724 /* select data transfer mode */ 1725 if (!of_property_read_string(child, "ti,nand-xfer-type", &s)) { 1726 for (i = 0; i < ARRAY_SIZE(nand_xfer_types); i++) { 1727 if (!strcasecmp(s, nand_xfer_types[i])) { 1728 info->xfer_type = i; 1729 return 0; 1730 } 1731 } 1732 1733 dev_err(dev, "unrecognized value for ti,nand-xfer-type\n"); 1734 return -EINVAL; 1735 } 1736 1737 return 0; 1738} 1739 1740static int omap_ooblayout_ecc(struct mtd_info *mtd, int section, 1741 struct mtd_oob_region *oobregion) 1742{ 1743 struct omap_nand_info *info = mtd_to_omap(mtd); 1744 struct nand_chip *chip = &info->nand; 1745 int off = BBM_LEN; 1746 1747 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW && 1748 !(chip->options & NAND_BUSWIDTH_16)) 1749 off = 1; 1750 1751 if (section) 1752 return -ERANGE; 1753 1754 oobregion->offset = off; 1755 oobregion->length = chip->ecc.total; 1756 1757 return 0; 1758} 1759 1760static int omap_ooblayout_free(struct mtd_info *mtd, int section, 1761 struct mtd_oob_region *oobregion) 1762{ 1763 struct omap_nand_info *info = mtd_to_omap(mtd); 1764 struct nand_chip *chip = &info->nand; 1765 int off = BBM_LEN; 1766 1767 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW && 1768 !(chip->options & NAND_BUSWIDTH_16)) 1769 off = 1; 1770 1771 if (section) 1772 return -ERANGE; 1773 1774 off += chip->ecc.total; 1775 if (off >= mtd->oobsize) 1776 return -ERANGE; 1777 1778 oobregion->offset = off; 1779 oobregion->length = mtd->oobsize - off; 1780 1781 return 0; 1782} 1783 1784static const struct mtd_ooblayout_ops omap_ooblayout_ops = { 1785 .ecc = omap_ooblayout_ecc, 1786 .free = omap_ooblayout_free, 1787}; 1788 1789static int omap_sw_ooblayout_ecc(struct mtd_info *mtd, int section, 1790 struct mtd_oob_region *oobregion) 1791{ 1792 struct nand_device *nand = mtd_to_nanddev(mtd); 1793 unsigned int nsteps = nanddev_get_ecc_nsteps(nand); 1794 unsigned int ecc_bytes = nanddev_get_ecc_bytes_per_step(nand); 1795 int off = BBM_LEN; 1796 1797 if (section >= nsteps) 1798 return -ERANGE; 1799 1800 /* 1801 * When SW correction is employed, one OMAP specific marker byte is 1802 * reserved after each ECC step. 1803 */ 1804 oobregion->offset = off + (section * (ecc_bytes + 1)); 1805 oobregion->length = ecc_bytes; 1806 1807 return 0; 1808} 1809 1810static int omap_sw_ooblayout_free(struct mtd_info *mtd, int section, 1811 struct mtd_oob_region *oobregion) 1812{ 1813 struct nand_device *nand = mtd_to_nanddev(mtd); 1814 unsigned int nsteps = nanddev_get_ecc_nsteps(nand); 1815 unsigned int ecc_bytes = nanddev_get_ecc_bytes_per_step(nand); 1816 int off = BBM_LEN; 1817 1818 if (section) 1819 return -ERANGE; 1820 1821 /* 1822 * When SW correction is employed, one OMAP specific marker byte is 1823 * reserved after each ECC step. 1824 */ 1825 off += ((ecc_bytes + 1) * nsteps); 1826 if (off >= mtd->oobsize) 1827 return -ERANGE; 1828 1829 oobregion->offset = off; 1830 oobregion->length = mtd->oobsize - off; 1831 1832 return 0; 1833} 1834 1835static const struct mtd_ooblayout_ops omap_sw_ooblayout_ops = { 1836 .ecc = omap_sw_ooblayout_ecc, 1837 .free = omap_sw_ooblayout_free, 1838}; 1839 1840static int omap_nand_attach_chip(struct nand_chip *chip) 1841{ 1842 struct mtd_info *mtd = nand_to_mtd(chip); 1843 struct omap_nand_info *info = mtd_to_omap(mtd); 1844 struct device *dev = &info->pdev->dev; 1845 int min_oobbytes = BBM_LEN; 1846 int elm_bch_strength = -1; 1847 int oobbytes_per_step; 1848 dma_cap_mask_t mask; 1849 int err; 1850 1851 if (chip->bbt_options & NAND_BBT_USE_FLASH) 1852 chip->bbt_options |= NAND_BBT_NO_OOB; 1853 else 1854 chip->options |= NAND_SKIP_BBTSCAN; 1855 1856 /* Re-populate low-level callbacks based on xfer modes */ 1857 switch (info->xfer_type) { 1858 case NAND_OMAP_PREFETCH_POLLED: 1859 info->data_in = omap_nand_data_in_pref; 1860 info->data_out = omap_nand_data_out_pref; 1861 break; 1862 1863 case NAND_OMAP_POLLED: 1864 /* Use nand_base defaults for {read,write}_buf */ 1865 break; 1866 1867 case NAND_OMAP_PREFETCH_DMA: 1868 dma_cap_zero(mask); 1869 dma_cap_set(DMA_SLAVE, mask); 1870 info->dma = dma_request_chan(dev->parent, "rxtx"); 1871 1872 if (IS_ERR(info->dma)) { 1873 dev_err(dev, "DMA engine request failed\n"); 1874 return PTR_ERR(info->dma); 1875 } else { 1876 struct dma_slave_config cfg; 1877 1878 memset(&cfg, 0, sizeof(cfg)); 1879 cfg.src_addr = info->phys_base; 1880 cfg.dst_addr = info->phys_base; 1881 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 1882 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 1883 cfg.src_maxburst = 16; 1884 cfg.dst_maxburst = 16; 1885 err = dmaengine_slave_config(info->dma, &cfg); 1886 if (err) { 1887 dev_err(dev, 1888 "DMA engine slave config failed: %d\n", 1889 err); 1890 return err; 1891 } 1892 1893 info->data_in = omap_nand_data_in_dma_pref; 1894 info->data_out = omap_nand_data_out_dma_pref; 1895 } 1896 break; 1897 1898 case NAND_OMAP_PREFETCH_IRQ: 1899 info->gpmc_irq_fifo = platform_get_irq(info->pdev, 0); 1900 if (info->gpmc_irq_fifo < 0) 1901 return info->gpmc_irq_fifo; 1902 err = devm_request_irq(dev, info->gpmc_irq_fifo, 1903 omap_nand_irq, IRQF_SHARED, 1904 "gpmc-nand-fifo", info); 1905 if (err) { 1906 dev_err(dev, "Requesting IRQ %d, error %d\n", 1907 info->gpmc_irq_fifo, err); 1908 info->gpmc_irq_fifo = 0; 1909 return err; 1910 } 1911 1912 info->gpmc_irq_count = platform_get_irq(info->pdev, 1); 1913 if (info->gpmc_irq_count < 0) 1914 return info->gpmc_irq_count; 1915 err = devm_request_irq(dev, info->gpmc_irq_count, 1916 omap_nand_irq, IRQF_SHARED, 1917 "gpmc-nand-count", info); 1918 if (err) { 1919 dev_err(dev, "Requesting IRQ %d, error %d\n", 1920 info->gpmc_irq_count, err); 1921 info->gpmc_irq_count = 0; 1922 return err; 1923 } 1924 1925 info->data_in = omap_nand_data_in_irq_pref; 1926 info->data_out = omap_nand_data_out_irq_pref; 1927 break; 1928 1929 default: 1930 dev_err(dev, "xfer_type %d not supported!\n", info->xfer_type); 1931 return -EINVAL; 1932 } 1933 1934 if (!omap2_nand_ecc_check(info)) 1935 return -EINVAL; 1936 1937 /* 1938 * Bail out earlier to let NAND_ECC_ENGINE_TYPE_SOFT code create its own 1939 * ooblayout instead of using ours. 1940 */ 1941 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_SW) { 1942 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; 1943 chip->ecc.algo = NAND_ECC_ALGO_HAMMING; 1944 return 0; 1945 } 1946 1947 /* Populate MTD interface based on ECC scheme */ 1948 switch (info->ecc_opt) { 1949 case OMAP_ECC_HAM1_CODE_HW: 1950 dev_info(dev, "nand: using OMAP_ECC_HAM1_CODE_HW\n"); 1951 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 1952 chip->ecc.bytes = 3; 1953 chip->ecc.size = 512; 1954 chip->ecc.strength = 1; 1955 chip->ecc.calculate = omap_calculate_ecc; 1956 chip->ecc.hwctl = omap_enable_hwecc; 1957 chip->ecc.correct = omap_correct_data; 1958 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 1959 oobbytes_per_step = chip->ecc.bytes; 1960 1961 if (!(chip->options & NAND_BUSWIDTH_16)) 1962 min_oobbytes = 1; 1963 1964 break; 1965 1966 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW: 1967 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW_DETECTION_SW\n"); 1968 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 1969 chip->ecc.size = 512; 1970 chip->ecc.bytes = 7; 1971 chip->ecc.strength = 4; 1972 chip->ecc.hwctl = omap_enable_hwecc_bch; 1973 chip->ecc.correct = rawnand_sw_bch_correct; 1974 chip->ecc.calculate = omap_calculate_ecc_bch_sw; 1975 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops); 1976 /* Reserve one byte for the OMAP marker */ 1977 oobbytes_per_step = chip->ecc.bytes + 1; 1978 /* Software BCH library is used for locating errors */ 1979 err = rawnand_sw_bch_init(chip); 1980 if (err) { 1981 dev_err(dev, "Unable to use BCH library\n"); 1982 goto err_put_elm_dev; 1983 } 1984 break; 1985 1986 case OMAP_ECC_BCH4_CODE_HW: 1987 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW ECC scheme\n"); 1988 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 1989 chip->ecc.size = 512; 1990 /* 14th bit is kept reserved for ROM-code compatibility */ 1991 chip->ecc.bytes = 7 + 1; 1992 chip->ecc.strength = 4; 1993 chip->ecc.hwctl = omap_enable_hwecc_bch; 1994 chip->ecc.correct = omap_elm_correct_data; 1995 chip->ecc.read_page = omap_read_page_bch; 1996 chip->ecc.write_page = omap_write_page_bch; 1997 chip->ecc.write_subpage = omap_write_subpage_bch; 1998 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 1999 oobbytes_per_step = chip->ecc.bytes; 2000 elm_bch_strength = BCH4_ECC; 2001 break; 2002 2003 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: 2004 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n"); 2005 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2006 chip->ecc.size = 512; 2007 chip->ecc.bytes = 13; 2008 chip->ecc.strength = 8; 2009 chip->ecc.hwctl = omap_enable_hwecc_bch; 2010 chip->ecc.correct = rawnand_sw_bch_correct; 2011 chip->ecc.calculate = omap_calculate_ecc_bch_sw; 2012 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops); 2013 /* Reserve one byte for the OMAP marker */ 2014 oobbytes_per_step = chip->ecc.bytes + 1; 2015 /* Software BCH library is used for locating errors */ 2016 err = rawnand_sw_bch_init(chip); 2017 if (err) { 2018 dev_err(dev, "unable to use BCH library\n"); 2019 goto err_put_elm_dev; 2020 } 2021 break; 2022 2023 case OMAP_ECC_BCH8_CODE_HW: 2024 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW ECC scheme\n"); 2025 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2026 chip->ecc.size = 512; 2027 /* 14th bit is kept reserved for ROM-code compatibility */ 2028 chip->ecc.bytes = 13 + 1; 2029 chip->ecc.strength = 8; 2030 chip->ecc.hwctl = omap_enable_hwecc_bch; 2031 chip->ecc.correct = omap_elm_correct_data; 2032 chip->ecc.read_page = omap_read_page_bch; 2033 chip->ecc.write_page = omap_write_page_bch; 2034 chip->ecc.write_subpage = omap_write_subpage_bch; 2035 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2036 oobbytes_per_step = chip->ecc.bytes; 2037 elm_bch_strength = BCH8_ECC; 2038 break; 2039 2040 case OMAP_ECC_BCH16_CODE_HW: 2041 pr_info("Using OMAP_ECC_BCH16_CODE_HW ECC scheme\n"); 2042 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; 2043 chip->ecc.size = 512; 2044 chip->ecc.bytes = 26; 2045 chip->ecc.strength = 16; 2046 chip->ecc.hwctl = omap_enable_hwecc_bch; 2047 chip->ecc.correct = omap_elm_correct_data; 2048 chip->ecc.read_page = omap_read_page_bch; 2049 chip->ecc.write_page = omap_write_page_bch; 2050 chip->ecc.write_subpage = omap_write_subpage_bch; 2051 mtd_set_ooblayout(mtd, &omap_ooblayout_ops); 2052 oobbytes_per_step = chip->ecc.bytes; 2053 elm_bch_strength = BCH16_ECC; 2054 break; 2055 default: 2056 dev_err(dev, "Invalid or unsupported ECC scheme\n"); 2057 err = -EINVAL; 2058 goto err_put_elm_dev; 2059 } 2060 2061 if (elm_bch_strength >= 0) { 2062 chip->ecc.steps = mtd->writesize / chip->ecc.size; 2063 info->neccpg = chip->ecc.steps / ERROR_VECTOR_MAX; 2064 if (info->neccpg) { 2065 info->nsteps_per_eccpg = ERROR_VECTOR_MAX; 2066 } else { 2067 info->neccpg = 1; 2068 info->nsteps_per_eccpg = chip->ecc.steps; 2069 } 2070 info->eccpg_size = info->nsteps_per_eccpg * chip->ecc.size; 2071 info->eccpg_bytes = info->nsteps_per_eccpg * chip->ecc.bytes; 2072 2073 err = elm_config(info->elm_dev, elm_bch_strength, 2074 info->nsteps_per_eccpg, chip->ecc.size, 2075 chip->ecc.bytes); 2076 if (err < 0) 2077 goto err_put_elm_dev; 2078 } 2079 2080 /* Check if NAND device's OOB is enough to store ECC signatures */ 2081 min_oobbytes += (oobbytes_per_step * 2082 (mtd->writesize / chip->ecc.size)); 2083 if (mtd->oobsize < min_oobbytes) { 2084 dev_err(dev, 2085 "Not enough OOB bytes: required = %d, available=%d\n", 2086 min_oobbytes, mtd->oobsize); 2087 err = -EINVAL; 2088 goto err_put_elm_dev; 2089 } 2090 2091 return 0; 2092 2093err_put_elm_dev: 2094 put_device(info->elm_dev); 2095 2096 return err; 2097} 2098 2099static void omap_nand_detach_chip(struct nand_chip *chip) 2100{ 2101 struct mtd_info *mtd = nand_to_mtd(chip); 2102 struct omap_nand_info *info = mtd_to_omap(mtd); 2103 2104 put_device(info->elm_dev); 2105} 2106 2107static void omap_nand_data_in(struct nand_chip *chip, void *buf, 2108 unsigned int len, bool force_8bit) 2109{ 2110 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 2111 u32 alignment = ((uintptr_t)buf | len) & 3; 2112 2113 if (force_8bit || (alignment & 1)) 2114 ioread8_rep(info->fifo, buf, len); 2115 else if (alignment & 3) 2116 ioread16_rep(info->fifo, buf, len >> 1); 2117 else 2118 ioread32_rep(info->fifo, buf, len >> 2); 2119} 2120 2121static void omap_nand_data_out(struct nand_chip *chip, 2122 const void *buf, unsigned int len, 2123 bool force_8bit) 2124{ 2125 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 2126 u32 alignment = ((uintptr_t)buf | len) & 3; 2127 2128 if (force_8bit || (alignment & 1)) 2129 iowrite8_rep(info->fifo, buf, len); 2130 else if (alignment & 3) 2131 iowrite16_rep(info->fifo, buf, len >> 1); 2132 else 2133 iowrite32_rep(info->fifo, buf, len >> 2); 2134} 2135 2136static int omap_nand_exec_instr(struct nand_chip *chip, 2137 const struct nand_op_instr *instr) 2138{ 2139 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip)); 2140 unsigned int i; 2141 int ret; 2142 2143 switch (instr->type) { 2144 case NAND_OP_CMD_INSTR: 2145 iowrite8(instr->ctx.cmd.opcode, 2146 info->reg.gpmc_nand_command); 2147 break; 2148 2149 case NAND_OP_ADDR_INSTR: 2150 for (i = 0; i < instr->ctx.addr.naddrs; i++) { 2151 iowrite8(instr->ctx.addr.addrs[i], 2152 info->reg.gpmc_nand_address); 2153 } 2154 break; 2155 2156 case NAND_OP_DATA_IN_INSTR: 2157 info->data_in(chip, instr->ctx.data.buf.in, 2158 instr->ctx.data.len, 2159 instr->ctx.data.force_8bit); 2160 break; 2161 2162 case NAND_OP_DATA_OUT_INSTR: 2163 info->data_out(chip, instr->ctx.data.buf.out, 2164 instr->ctx.data.len, 2165 instr->ctx.data.force_8bit); 2166 break; 2167 2168 case NAND_OP_WAITRDY_INSTR: 2169 ret = info->ready_gpiod ? 2170 nand_gpio_waitrdy(chip, info->ready_gpiod, instr->ctx.waitrdy.timeout_ms) : 2171 nand_soft_waitrdy(chip, instr->ctx.waitrdy.timeout_ms); 2172 if (ret) 2173 return ret; 2174 break; 2175 } 2176 2177 if (instr->delay_ns) 2178 ndelay(instr->delay_ns); 2179 2180 return 0; 2181} 2182 2183static int omap_nand_exec_op(struct nand_chip *chip, 2184 const struct nand_operation *op, 2185 bool check_only) 2186{ 2187 unsigned int i; 2188 2189 if (check_only) 2190 return 0; 2191 2192 for (i = 0; i < op->ninstrs; i++) { 2193 int ret; 2194 2195 ret = omap_nand_exec_instr(chip, &op->instrs[i]); 2196 if (ret) 2197 return ret; 2198 } 2199 2200 return 0; 2201} 2202 2203static const struct nand_controller_ops omap_nand_controller_ops = { 2204 .attach_chip = omap_nand_attach_chip, 2205 .detach_chip = omap_nand_detach_chip, 2206 .exec_op = omap_nand_exec_op, 2207}; 2208 2209/* Shared among all NAND instances to synchronize access to the ECC Engine */ 2210static struct nand_controller omap_gpmc_controller; 2211static bool omap_gpmc_controller_initialized; 2212 2213static int omap_nand_probe(struct platform_device *pdev) 2214{ 2215 struct omap_nand_info *info; 2216 struct mtd_info *mtd; 2217 struct nand_chip *nand_chip; 2218 int err; 2219 struct resource *res; 2220 struct device *dev = &pdev->dev; 2221 void __iomem *vaddr; 2222 2223 info = devm_kzalloc(&pdev->dev, sizeof(struct omap_nand_info), 2224 GFP_KERNEL); 2225 if (!info) 2226 return -ENOMEM; 2227 2228 info->pdev = pdev; 2229 2230 err = omap_get_dt_info(dev, info); 2231 if (err) 2232 return err; 2233 2234 info->ops = gpmc_omap_get_nand_ops(&info->reg, info->gpmc_cs); 2235 if (!info->ops) { 2236 dev_err(&pdev->dev, "Failed to get GPMC->NAND interface\n"); 2237 return -ENODEV; 2238 } 2239 2240 nand_chip = &info->nand; 2241 mtd = nand_to_mtd(nand_chip); 2242 mtd->dev.parent = &pdev->dev; 2243 nand_set_flash_node(nand_chip, dev->of_node); 2244 2245 if (!mtd->name) { 2246 mtd->name = devm_kasprintf(&pdev->dev, GFP_KERNEL, 2247 "omap2-nand.%d", info->gpmc_cs); 2248 if (!mtd->name) { 2249 dev_err(&pdev->dev, "Failed to set MTD name\n"); 2250 return -ENOMEM; 2251 } 2252 } 2253 2254 vaddr = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 2255 if (IS_ERR(vaddr)) 2256 return PTR_ERR(vaddr); 2257 2258 info->fifo = vaddr; 2259 info->phys_base = res->start; 2260 2261 if (!omap_gpmc_controller_initialized) { 2262 omap_gpmc_controller.ops = &omap_nand_controller_ops; 2263 nand_controller_init(&omap_gpmc_controller); 2264 omap_gpmc_controller_initialized = true; 2265 } 2266 2267 nand_chip->controller = &omap_gpmc_controller; 2268 2269 info->ready_gpiod = devm_gpiod_get_optional(&pdev->dev, "rb", 2270 GPIOD_IN); 2271 if (IS_ERR(info->ready_gpiod)) { 2272 dev_err(dev, "failed to get ready gpio\n"); 2273 return PTR_ERR(info->ready_gpiod); 2274 } 2275 2276 if (info->flash_bbt) 2277 nand_chip->bbt_options |= NAND_BBT_USE_FLASH; 2278 2279 /* default operations */ 2280 info->data_in = omap_nand_data_in; 2281 info->data_out = omap_nand_data_out; 2282 2283 err = nand_scan(nand_chip, 1); 2284 if (err) 2285 goto return_error; 2286 2287 err = mtd_device_register(mtd, NULL, 0); 2288 if (err) 2289 goto cleanup_nand; 2290 2291 platform_set_drvdata(pdev, mtd); 2292 2293 return 0; 2294 2295cleanup_nand: 2296 nand_cleanup(nand_chip); 2297 2298return_error: 2299 if (!IS_ERR_OR_NULL(info->dma)) 2300 dma_release_channel(info->dma); 2301 2302 rawnand_sw_bch_cleanup(nand_chip); 2303 2304 return err; 2305} 2306 2307static void omap_nand_remove(struct platform_device *pdev) 2308{ 2309 struct mtd_info *mtd = platform_get_drvdata(pdev); 2310 struct nand_chip *nand_chip = mtd_to_nand(mtd); 2311 struct omap_nand_info *info = mtd_to_omap(mtd); 2312 2313 rawnand_sw_bch_cleanup(nand_chip); 2314 2315 if (info->dma) 2316 dma_release_channel(info->dma); 2317 WARN_ON(mtd_device_unregister(mtd)); 2318 nand_cleanup(nand_chip); 2319} 2320 2321/* omap_nand_ids defined in linux/platform_data/mtd-nand-omap2.h */ 2322MODULE_DEVICE_TABLE(of, omap_nand_ids); 2323 2324static struct platform_driver omap_nand_driver = { 2325 .probe = omap_nand_probe, 2326 .remove = omap_nand_remove, 2327 .driver = { 2328 .name = DRIVER_NAME, 2329 .of_match_table = omap_nand_ids, 2330 }, 2331}; 2332 2333module_platform_driver(omap_nand_driver); 2334 2335MODULE_LICENSE("GPL"); 2336MODULE_DESCRIPTION("Glue layer for NAND flash on TI OMAP boards");
~/
Read-only file preview.