rockchip-nand-controller.c

   1// SPDX-License-Identifier: GPL-2.0 OR MIT
   2/*
   3 * Rockchip NAND Flash controller driver.
   4 * Copyright (C) 2020 Rockchip Inc.
   5 * Author: Yifeng Zhao <yifeng.zhao@rock-chips.com>
   6 */
   7
   8#include <linux/clk.h>
   9#include <linux/delay.h>
  10#include <linux/dma-mapping.h>
  11#include <linux/dmaengine.h>
  12#include <linux/interrupt.h>
  13#include <linux/iopoll.h>
  14#include <linux/module.h>
  15#include <linux/mtd/mtd.h>
  16#include <linux/mtd/rawnand.h>
  17#include <linux/of.h>
  18#include <linux/platform_device.h>
  19#include <linux/slab.h>
  20
  21/*
  22 * NFC Page Data Layout:
  23 *	1024 bytes data + 4Bytes sys data + 28Bytes~124Bytes ECC data +
  24 *	1024 bytes data + 4Bytes sys data + 28Bytes~124Bytes ECC data +
  25 *	......
  26 * NAND Page Data Layout:
  27 *	1024 * n data + m Bytes oob
  28 * Original Bad Block Mask Location:
  29 *	First byte of oob(spare).
  30 * nand_chip->oob_poi data layout:
  31 *	4Bytes sys data + .... + 4Bytes sys data + ECC data.
  32 */
  33
  34/* NAND controller register definition */
  35#define NFC_READ			(0)
  36#define NFC_WRITE			(1)
  37
  38#define NFC_FMCTL			(0x00)
  39#define   FMCTL_CE_SEL_M		0xFF
  40#define   FMCTL_CE_SEL(x)		(1 << (x))
  41#define   FMCTL_WP			BIT(8)
  42#define   FMCTL_RDY			BIT(9)
  43
  44#define NFC_FMWAIT			(0x04)
  45#define   FLCTL_RST			BIT(0)
  46#define   FLCTL_WR			(1)	/* 0: read, 1: write */
  47#define   FLCTL_XFER_ST			BIT(2)
  48#define   FLCTL_XFER_EN			BIT(3)
  49#define   FLCTL_ACORRECT		BIT(10) /* Auto correct error bits. */
  50#define   FLCTL_XFER_READY		BIT(20)
  51#define   FLCTL_XFER_SECTOR		(22)
  52#define   FLCTL_TOG_FIX			BIT(29)
  53
  54#define   BCHCTL_BANK_M			(7 << 5)
  55#define   BCHCTL_BANK			(5)
  56
  57#define   DMA_ST			BIT(0)
  58#define   DMA_WR			(1)	/* 0: write, 1: read */
  59#define   DMA_EN			BIT(2)
  60#define   DMA_AHB_SIZE			(3)	/* 0: 1, 1: 2, 2: 4 */
  61#define   DMA_BURST_SIZE		(6)	/* 0: 1, 3: 4, 5: 8, 7: 16 */
  62#define   DMA_INC_NUM			(9)	/* 1 - 16 */
  63
  64#define ECC_ERR_CNT(x, e) ((((x) >> (e).low) & (e).low_mask) |\
  65	  (((x) >> (e).high) & (e).high_mask) << (e).low_bn)
  66#define   INT_DMA			BIT(0)
  67#define NFC_BANK			(0x800)
  68#define NFC_BANK_STEP			(0x100)
  69#define   BANK_DATA			(0x00)
  70#define   BANK_ADDR			(0x04)
  71#define   BANK_CMD			(0x08)
  72#define NFC_SRAM0			(0x1000)
  73#define NFC_SRAM1			(0x1400)
  74#define NFC_SRAM_SIZE			(0x400)
  75#define NFC_TIMEOUT			(500000)
  76#define NFC_MAX_OOB_PER_STEP		128
  77#define NFC_MIN_OOB_PER_STEP		64
  78#define MAX_DATA_SIZE			0xFFFC
  79#define MAX_ADDRESS_CYC			6
  80#define NFC_ECC_MAX_MODES		4
  81#define NFC_MAX_NSELS			(8) /* Some Socs only have 1 or 2 CSs. */
  82#define NFC_SYS_DATA_SIZE		(4) /* 4 bytes sys data in oob pre 1024 data.*/
  83#define RK_DEFAULT_CLOCK_RATE		(150 * 1000 * 1000) /* 150 Mhz */
  84#define ACCTIMING(csrw, rwpw, rwcs)	((csrw) << 12 | (rwpw) << 5 | (rwcs))
  85
  86enum nfc_type {
  87	NFC_V6,
  88	NFC_V8,
  89	NFC_V9,
  90};
  91
  92/**
  93 * struct rk_ecc_cnt_status: represent a ecc status data.
  94 * @err_flag_bit: error flag bit index at register.
  95 * @low: ECC count low bit index at register.
  96 * @low_mask: mask bit.
  97 * @low_bn: ECC count low bit number.
  98 * @high: ECC count high bit index at register.
  99 * @high_mask: mask bit
 100 */
 101struct rk_ecc_cnt_status {
 102	u8 err_flag_bit;
 103	u8 low;
 104	u8 low_mask;
 105	u8 low_bn;
 106	u8 high;
 107	u8 high_mask;
 108};
 109
 110/**
 111 * struct nfc_cfg: Rockchip NAND controller configuration
 112 * @type: NFC version
 113 * @ecc_strengths: ECC strengths
 114 * @ecc_cfgs: ECC config values
 115 * @flctl_off: FLCTL register offset
 116 * @bchctl_off: BCHCTL register offset
 117 * @dma_data_buf_off: DMA_DATA_BUF register offset
 118 * @dma_oob_buf_off: DMA_OOB_BUF register offset
 119 * @dma_cfg_off: DMA_CFG register offset
 120 * @dma_st_off: DMA_ST register offset
 121 * @bch_st_off: BCG_ST register offset
 122 * @randmz_off: RANDMZ register offset
 123 * @int_en_off: interrupt enable register offset
 124 * @int_clr_off: interrupt clean register offset
 125 * @int_st_off: interrupt status register offset
 126 * @oob0_off: oob0 register offset
 127 * @oob1_off: oob1 register offset
 128 * @ecc0: represent ECC0 status data
 129 * @ecc1: represent ECC1 status data
 130 */
 131struct nfc_cfg {
 132	enum nfc_type type;
 133	u8 ecc_strengths[NFC_ECC_MAX_MODES];
 134	u32 ecc_cfgs[NFC_ECC_MAX_MODES];
 135	u32 flctl_off;
 136	u32 bchctl_off;
 137	u32 dma_cfg_off;
 138	u32 dma_data_buf_off;
 139	u32 dma_oob_buf_off;
 140	u32 dma_st_off;
 141	u32 bch_st_off;
 142	u32 randmz_off;
 143	u32 int_en_off;
 144	u32 int_clr_off;
 145	u32 int_st_off;
 146	u32 oob0_off;
 147	u32 oob1_off;
 148	struct rk_ecc_cnt_status ecc0;
 149	struct rk_ecc_cnt_status ecc1;
 150};
 151
 152struct rk_nfc_nand_chip {
 153	struct list_head node;
 154	struct nand_chip chip;
 155
 156	u16 boot_blks;
 157	u16 metadata_size;
 158	u32 boot_ecc;
 159	u32 timing;
 160
 161	u8 nsels;
 162	u8 sels[] __counted_by(nsels);
 163};
 164
 165struct rk_nfc {
 166	struct nand_controller controller;
 167	const struct nfc_cfg *cfg;
 168	struct device *dev;
 169
 170	struct clk *nfc_clk;
 171	struct clk *ahb_clk;
 172	void __iomem *regs;
 173
 174	u32 selected_bank;
 175	u32 band_offset;
 176	u32 cur_ecc;
 177	u32 cur_timing;
 178
 179	struct completion done;
 180	struct list_head chips;
 181
 182	u8 *page_buf;
 183	u32 *oob_buf;
 184	u32 page_buf_size;
 185	u32 oob_buf_size;
 186
 187	unsigned long assigned_cs;
 188};
 189
 190static inline struct rk_nfc_nand_chip *rk_nfc_to_rknand(struct nand_chip *chip)
 191{
 192	return container_of(chip, struct rk_nfc_nand_chip, chip);
 193}
 194
 195static inline u8 *rk_nfc_buf_to_data_ptr(struct nand_chip *chip, const u8 *p, int i)
 196{
 197	return (u8 *)p + i * chip->ecc.size;
 198}
 199
 200static inline u8 *rk_nfc_buf_to_oob_ptr(struct nand_chip *chip, int i)
 201{
 202	u8 *poi;
 203
 204	poi = chip->oob_poi + i * NFC_SYS_DATA_SIZE;
 205
 206	return poi;
 207}
 208
 209static inline u8 *rk_nfc_buf_to_oob_ecc_ptr(struct nand_chip *chip, int i)
 210{
 211	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 212	u8 *poi;
 213
 214	poi = chip->oob_poi + rknand->metadata_size + chip->ecc.bytes * i;
 215
 216	return poi;
 217}
 218
 219static inline int rk_nfc_data_len(struct nand_chip *chip)
 220{
 221	return chip->ecc.size + chip->ecc.bytes + NFC_SYS_DATA_SIZE;
 222}
 223
 224static inline u8 *rk_nfc_data_ptr(struct nand_chip *chip, int i)
 225{
 226	struct rk_nfc *nfc = nand_get_controller_data(chip);
 227
 228	return nfc->page_buf + i * rk_nfc_data_len(chip);
 229}
 230
 231static inline u8 *rk_nfc_oob_ptr(struct nand_chip *chip, int i)
 232{
 233	struct rk_nfc *nfc = nand_get_controller_data(chip);
 234
 235	return nfc->page_buf + i * rk_nfc_data_len(chip) + chip->ecc.size;
 236}
 237
 238static int rk_nfc_hw_ecc_setup(struct nand_chip *chip, u32 strength)
 239{
 240	struct rk_nfc *nfc = nand_get_controller_data(chip);
 241	u32 reg, i;
 242
 243	for (i = 0; i < NFC_ECC_MAX_MODES; i++) {
 244		if (strength == nfc->cfg->ecc_strengths[i]) {
 245			reg = nfc->cfg->ecc_cfgs[i];
 246			break;
 247		}
 248	}
 249
 250	if (i >= NFC_ECC_MAX_MODES)
 251		return -EINVAL;
 252
 253	writel(reg, nfc->regs + nfc->cfg->bchctl_off);
 254
 255	/* Save chip ECC setting */
 256	nfc->cur_ecc = strength;
 257
 258	return 0;
 259}
 260
 261static void rk_nfc_select_chip(struct nand_chip *chip, int cs)
 262{
 263	struct rk_nfc *nfc = nand_get_controller_data(chip);
 264	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 265	struct nand_ecc_ctrl *ecc = &chip->ecc;
 266	u32 val;
 267
 268	if (cs < 0) {
 269		nfc->selected_bank = -1;
 270		/* Deselect the currently selected target. */
 271		val = readl_relaxed(nfc->regs + NFC_FMCTL);
 272		val &= ~FMCTL_CE_SEL_M;
 273		writel(val, nfc->regs + NFC_FMCTL);
 274		return;
 275	}
 276
 277	nfc->selected_bank = rknand->sels[cs];
 278	nfc->band_offset = NFC_BANK + nfc->selected_bank * NFC_BANK_STEP;
 279
 280	val = readl_relaxed(nfc->regs + NFC_FMCTL);
 281	val &= ~FMCTL_CE_SEL_M;
 282	val |= FMCTL_CE_SEL(nfc->selected_bank);
 283
 284	writel(val, nfc->regs + NFC_FMCTL);
 285
 286	/*
 287	 * Compare current chip timing with selected chip timing and
 288	 * change if needed.
 289	 */
 290	if (nfc->cur_timing != rknand->timing) {
 291		writel(rknand->timing, nfc->regs + NFC_FMWAIT);
 292		nfc->cur_timing = rknand->timing;
 293	}
 294
 295	/*
 296	 * Compare current chip ECC setting with selected chip ECC setting and
 297	 * change if needed.
 298	 */
 299	if (nfc->cur_ecc != ecc->strength)
 300		rk_nfc_hw_ecc_setup(chip, ecc->strength);
 301}
 302
 303static inline int rk_nfc_wait_ioready(struct rk_nfc *nfc)
 304{
 305	int rc;
 306	u32 val;
 307
 308	rc = readl_relaxed_poll_timeout(nfc->regs + NFC_FMCTL, val,
 309					val & FMCTL_RDY, 10, NFC_TIMEOUT);
 310
 311	return rc;
 312}
 313
 314static void rk_nfc_read_buf(struct rk_nfc *nfc, u8 *buf, int len)
 315{
 316	int i;
 317
 318	for (i = 0; i < len; i++)
 319		buf[i] = readb_relaxed(nfc->regs + nfc->band_offset +
 320				       BANK_DATA);
 321}
 322
 323static void rk_nfc_write_buf(struct rk_nfc *nfc, const u8 *buf, int len)
 324{
 325	int i;
 326
 327	for (i = 0; i < len; i++)
 328		writeb(buf[i], nfc->regs + nfc->band_offset + BANK_DATA);
 329}
 330
 331static int rk_nfc_cmd(struct nand_chip *chip,
 332		      const struct nand_subop *subop)
 333{
 334	struct rk_nfc *nfc = nand_get_controller_data(chip);
 335	unsigned int i, j, remaining, start;
 336	int reg_offset = nfc->band_offset;
 337	u8 *inbuf = NULL;
 338	const u8 *outbuf;
 339	u32 cnt = 0;
 340	int ret = 0;
 341
 342	for (i = 0; i < subop->ninstrs; i++) {
 343		const struct nand_op_instr *instr = &subop->instrs[i];
 344
 345		switch (instr->type) {
 346		case NAND_OP_CMD_INSTR:
 347			writeb(instr->ctx.cmd.opcode,
 348			       nfc->regs + reg_offset + BANK_CMD);
 349			break;
 350
 351		case NAND_OP_ADDR_INSTR:
 352			remaining = nand_subop_get_num_addr_cyc(subop, i);
 353			start = nand_subop_get_addr_start_off(subop, i);
 354
 355			for (j = 0; j < 8 && j + start < remaining; j++)
 356				writeb(instr->ctx.addr.addrs[j + start],
 357				       nfc->regs + reg_offset + BANK_ADDR);
 358			break;
 359
 360		case NAND_OP_DATA_IN_INSTR:
 361		case NAND_OP_DATA_OUT_INSTR:
 362			start = nand_subop_get_data_start_off(subop, i);
 363			cnt = nand_subop_get_data_len(subop, i);
 364
 365			if (instr->type == NAND_OP_DATA_OUT_INSTR) {
 366				outbuf = instr->ctx.data.buf.out + start;
 367				rk_nfc_write_buf(nfc, outbuf, cnt);
 368			} else {
 369				inbuf = instr->ctx.data.buf.in + start;
 370				rk_nfc_read_buf(nfc, inbuf, cnt);
 371			}
 372			break;
 373
 374		case NAND_OP_WAITRDY_INSTR:
 375			if (rk_nfc_wait_ioready(nfc) < 0) {
 376				ret = -ETIMEDOUT;
 377				dev_err(nfc->dev, "IO not ready\n");
 378			}
 379			break;
 380		}
 381	}
 382
 383	return ret;
 384}
 385
 386static const struct nand_op_parser rk_nfc_op_parser = NAND_OP_PARSER(
 387	NAND_OP_PARSER_PATTERN(
 388		rk_nfc_cmd,
 389		NAND_OP_PARSER_PAT_CMD_ELEM(true),
 390		NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC),
 391		NAND_OP_PARSER_PAT_CMD_ELEM(true),
 392		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
 393		NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, MAX_DATA_SIZE)),
 394	NAND_OP_PARSER_PATTERN(
 395		rk_nfc_cmd,
 396		NAND_OP_PARSER_PAT_CMD_ELEM(true),
 397		NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC),
 398		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, MAX_DATA_SIZE),
 399		NAND_OP_PARSER_PAT_CMD_ELEM(true),
 400		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
 401);
 402
 403static int rk_nfc_exec_op(struct nand_chip *chip,
 404			  const struct nand_operation *op,
 405			  bool check_only)
 406{
 407	if (!check_only)
 408		rk_nfc_select_chip(chip, op->cs);
 409
 410	return nand_op_parser_exec_op(chip, &rk_nfc_op_parser, op,
 411				      check_only);
 412}
 413
 414static int rk_nfc_setup_interface(struct nand_chip *chip, int target,
 415				  const struct nand_interface_config *conf)
 416{
 417	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 418	struct rk_nfc *nfc = nand_get_controller_data(chip);
 419	const struct nand_sdr_timings *timings;
 420	u32 rate, tc2rw, trwpw, trw2c;
 421	u32 temp;
 422
 423	timings = nand_get_sdr_timings(conf);
 424	if (IS_ERR(timings))
 425		return -EOPNOTSUPP;
 426
 427	if (target < 0)
 428		return 0;
 429
 430	if (IS_ERR(nfc->nfc_clk))
 431		rate = clk_get_rate(nfc->ahb_clk);
 432	else
 433		rate = clk_get_rate(nfc->nfc_clk);
 434
 435	/* Turn clock rate into kHz. */
 436	rate /= 1000;
 437
 438	tc2rw = 1;
 439	trw2c = 1;
 440
 441	trwpw = max(timings->tWC_min, timings->tRC_min) / 1000;
 442	trwpw = DIV_ROUND_UP(trwpw * rate, 1000000);
 443
 444	temp = timings->tREA_max / 1000;
 445	temp = DIV_ROUND_UP(temp * rate, 1000000);
 446
 447	if (trwpw < temp)
 448		trwpw = temp;
 449
 450	/*
 451	 * ACCON: access timing control register
 452	 * -------------------------------------
 453	 * 31:18: reserved
 454	 * 17:12: csrw, clock cycles from the falling edge of CSn to the
 455	 *   falling edge of RDn or WRn
 456	 * 11:11: reserved
 457	 * 10:05: rwpw, the width of RDn or WRn in processor clock cycles
 458	 * 04:00: rwcs, clock cycles from the rising edge of RDn or WRn to the
 459	 *   rising edge of CSn
 460	 */
 461
 462	/* Save chip timing */
 463	rknand->timing = ACCTIMING(tc2rw, trwpw, trw2c);
 464
 465	return 0;
 466}
 467
 468static void rk_nfc_xfer_start(struct rk_nfc *nfc, u8 rw, u8 n_KB,
 469			      dma_addr_t dma_data, dma_addr_t dma_oob)
 470{
 471	u32 dma_reg, fl_reg, bch_reg;
 472
 473	dma_reg = DMA_ST | ((!rw) << DMA_WR) | DMA_EN | (2 << DMA_AHB_SIZE) |
 474	      (7 << DMA_BURST_SIZE) | (16 << DMA_INC_NUM);
 475
 476	fl_reg = (rw << FLCTL_WR) | FLCTL_XFER_EN | FLCTL_ACORRECT |
 477		 (n_KB << FLCTL_XFER_SECTOR) | FLCTL_TOG_FIX;
 478
 479	if (nfc->cfg->type == NFC_V6 || nfc->cfg->type == NFC_V8) {
 480		bch_reg = readl_relaxed(nfc->regs + nfc->cfg->bchctl_off);
 481		bch_reg = (bch_reg & (~BCHCTL_BANK_M)) |
 482			  (nfc->selected_bank << BCHCTL_BANK);
 483		writel(bch_reg, nfc->regs + nfc->cfg->bchctl_off);
 484	}
 485
 486	writel(dma_reg, nfc->regs + nfc->cfg->dma_cfg_off);
 487	writel((u32)dma_data, nfc->regs + nfc->cfg->dma_data_buf_off);
 488	writel((u32)dma_oob, nfc->regs + nfc->cfg->dma_oob_buf_off);
 489	writel(fl_reg, nfc->regs + nfc->cfg->flctl_off);
 490	fl_reg |= FLCTL_XFER_ST;
 491	writel(fl_reg, nfc->regs + nfc->cfg->flctl_off);
 492}
 493
 494static int rk_nfc_wait_for_xfer_done(struct rk_nfc *nfc)
 495{
 496	void __iomem *ptr;
 497	u32 reg;
 498
 499	ptr = nfc->regs + nfc->cfg->flctl_off;
 500
 501	return readl_relaxed_poll_timeout(ptr, reg,
 502					 reg & FLCTL_XFER_READY,
 503					 10, NFC_TIMEOUT);
 504}
 505
 506static int rk_nfc_write_page_raw(struct nand_chip *chip, const u8 *buf,
 507				 int oob_on, int page)
 508{
 509	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 510	struct rk_nfc *nfc = nand_get_controller_data(chip);
 511	struct mtd_info *mtd = nand_to_mtd(chip);
 512	struct nand_ecc_ctrl *ecc = &chip->ecc;
 513	int i, pages_per_blk;
 514
 515	pages_per_blk = mtd->erasesize / mtd->writesize;
 516	if ((chip->options & NAND_IS_BOOT_MEDIUM) &&
 517	    (page < (pages_per_blk * rknand->boot_blks)) &&
 518	    rknand->boot_ecc != ecc->strength) {
 519		/*
 520		 * There's currently no method to notify the MTD framework that
 521		 * a different ECC strength is in use for the boot blocks.
 522		 */
 523		return -EIO;
 524	}
 525
 526	if (!buf)
 527		memset(nfc->page_buf, 0xff, mtd->writesize + mtd->oobsize);
 528
 529	for (i = 0; i < ecc->steps; i++) {
 530		/* Copy data to the NFC buffer. */
 531		if (buf)
 532			memcpy(rk_nfc_data_ptr(chip, i),
 533			       rk_nfc_buf_to_data_ptr(chip, buf, i),
 534			       ecc->size);
 535		/*
 536		 * The first four bytes of OOB are reserved for the
 537		 * boot ROM. In some debugging cases, such as with a
 538		 * read, erase and write back test these 4 bytes stored
 539		 * in OOB also need to be written back.
 540		 *
 541		 * The function nand_block_bad detects bad blocks like:
 542		 *
 543		 * bad = chip->oob_poi[chip->badblockpos];
 544		 *
 545		 * chip->badblockpos == 0 for a large page NAND Flash,
 546		 * so chip->oob_poi[0] is the bad block mask (BBM).
 547		 *
 548		 * The OOB data layout on the NFC is:
 549		 *
 550		 *    PA0  PA1  PA2  PA3  | BBM OOB1 OOB2 OOB3 | ...
 551		 *
 552		 * or
 553		 *
 554		 *    0xFF 0xFF 0xFF 0xFF | BBM OOB1 OOB2 OOB3 | ...
 555		 *
 556		 * The code here just swaps the first 4 bytes with the last
 557		 * 4 bytes without losing any data.
 558		 *
 559		 * The chip->oob_poi data layout:
 560		 *
 561		 *    BBM  OOB1 OOB2 OOB3 |......|  PA0  PA1  PA2  PA3
 562		 *
 563		 * The rk_nfc_ooblayout_free() function already has reserved
 564		 * these 4 bytes together with 2 bytes for BBM
 565		 * by reducing it's length:
 566		 *
 567		 * oob_region->length = rknand->metadata_size - NFC_SYS_DATA_SIZE - 2;
 568		 */
 569		if (!i)
 570			memcpy(rk_nfc_oob_ptr(chip, i),
 571			       rk_nfc_buf_to_oob_ptr(chip, ecc->steps - 1),
 572			       NFC_SYS_DATA_SIZE);
 573		else
 574			memcpy(rk_nfc_oob_ptr(chip, i),
 575			       rk_nfc_buf_to_oob_ptr(chip, i - 1),
 576			       NFC_SYS_DATA_SIZE);
 577		/* Copy ECC data to the NFC buffer. */
 578		memcpy(rk_nfc_oob_ptr(chip, i) + NFC_SYS_DATA_SIZE,
 579		       rk_nfc_buf_to_oob_ecc_ptr(chip, i),
 580		       ecc->bytes);
 581	}
 582
 583	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 584	rk_nfc_write_buf(nfc, buf, mtd->writesize + mtd->oobsize);
 585	return nand_prog_page_end_op(chip);
 586}
 587
 588static int rk_nfc_write_page_hwecc(struct nand_chip *chip, const u8 *buf,
 589				   int oob_on, int page)
 590{
 591	struct mtd_info *mtd = nand_to_mtd(chip);
 592	struct rk_nfc *nfc = nand_get_controller_data(chip);
 593	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 594	struct nand_ecc_ctrl *ecc = &chip->ecc;
 595	int oob_step = (ecc->bytes > 60) ? NFC_MAX_OOB_PER_STEP :
 596			NFC_MIN_OOB_PER_STEP;
 597	int pages_per_blk = mtd->erasesize / mtd->writesize;
 598	int ret = 0, i, boot_rom_mode = 0;
 599	dma_addr_t dma_data, dma_oob;
 600	u32 tmp;
 601	u8 *oob;
 602
 603	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 604
 605	if (buf)
 606		memcpy(nfc->page_buf, buf, mtd->writesize);
 607	else
 608		memset(nfc->page_buf, 0xFF, mtd->writesize);
 609
 610	/*
 611	 * The first blocks (4, 8 or 16 depending on the device) are used
 612	 * by the boot ROM and the first 32 bits of OOB need to link to
 613	 * the next page address in the same block. We can't directly copy
 614	 * OOB data from the MTD framework, because this page address
 615	 * conflicts for example with the bad block marker (BBM),
 616	 * so we shift all OOB data including the BBM with 4 byte positions.
 617	 * As a consequence the OOB size available to the MTD framework is
 618	 * also reduced with 4 bytes.
 619	 *
 620	 *    PA0  PA1  PA2  PA3 | BBM OOB1 OOB2 OOB3 | ...
 621	 *
 622	 * If a NAND is not a boot medium or the page is not a boot block,
 623	 * the first 4 bytes are left untouched by writing 0xFF to them.
 624	 *
 625	 *   0xFF 0xFF 0xFF 0xFF | BBM OOB1 OOB2 OOB3 | ...
 626	 *
 627	 * The code here just swaps the first 4 bytes with the last
 628	 * 4 bytes without losing any data.
 629	 *
 630	 * The chip->oob_poi data layout:
 631	 *
 632	 *    BBM  OOB1 OOB2 OOB3 |......|  PA0  PA1  PA2  PA3
 633	 *
 634	 * Configure the ECC algorithm supported by the boot ROM.
 635	 */
 636	if ((page < (pages_per_blk * rknand->boot_blks)) &&
 637	    (chip->options & NAND_IS_BOOT_MEDIUM)) {
 638		boot_rom_mode = 1;
 639		if (rknand->boot_ecc != ecc->strength)
 640			rk_nfc_hw_ecc_setup(chip, rknand->boot_ecc);
 641	}
 642
 643	for (i = 0; i < ecc->steps; i++) {
 644		if (!i)
 645			oob = chip->oob_poi + (ecc->steps - 1) * NFC_SYS_DATA_SIZE;
 646		else
 647			oob = chip->oob_poi + (i - 1) * NFC_SYS_DATA_SIZE;
 648
 649		tmp = oob[0] | oob[1] << 8 | oob[2] << 16 | oob[3] << 24;
 650
 651		if (nfc->cfg->type == NFC_V9)
 652			nfc->oob_buf[i] = tmp;
 653		else
 654			nfc->oob_buf[i * (oob_step / 4)] = tmp;
 655	}
 656
 657	dma_data = dma_map_single(nfc->dev, (void *)nfc->page_buf,
 658				  mtd->writesize, DMA_TO_DEVICE);
 659	if (dma_mapping_error(nfc->dev, dma_data))
 660		return -ENOMEM;
 661
 662	dma_oob = dma_map_single(nfc->dev, nfc->oob_buf,
 663				 ecc->steps * oob_step,
 664				 DMA_TO_DEVICE);
 665	if (dma_mapping_error(nfc->dev, dma_oob)) {
 666		dma_unmap_single(nfc->dev, dma_data, mtd->writesize, DMA_TO_DEVICE);
 667		return -ENOMEM;
 668	}
 669
 670	reinit_completion(&nfc->done);
 671	writel(INT_DMA, nfc->regs + nfc->cfg->int_en_off);
 672
 673	rk_nfc_xfer_start(nfc, NFC_WRITE, ecc->steps, dma_data,
 674			  dma_oob);
 675	ret = wait_for_completion_timeout(&nfc->done,
 676					  msecs_to_jiffies(100));
 677	if (!ret)
 678		dev_warn(nfc->dev, "write: wait dma done timeout.\n");
 679	/*
 680	 * Whether the DMA transfer is completed or not. The driver
 681	 * needs to check the NFC`s status register to see if the data
 682	 * transfer was completed.
 683	 */
 684	ret = rk_nfc_wait_for_xfer_done(nfc);
 685
 686	dma_unmap_single(nfc->dev, dma_data, mtd->writesize,
 687			 DMA_TO_DEVICE);
 688	dma_unmap_single(nfc->dev, dma_oob, ecc->steps * oob_step,
 689			 DMA_TO_DEVICE);
 690
 691	if (boot_rom_mode && rknand->boot_ecc != ecc->strength)
 692		rk_nfc_hw_ecc_setup(chip, ecc->strength);
 693
 694	if (ret) {
 695		dev_err(nfc->dev, "write: wait transfer done timeout.\n");
 696		return -ETIMEDOUT;
 697	}
 698
 699	return nand_prog_page_end_op(chip);
 700}
 701
 702static int rk_nfc_write_oob(struct nand_chip *chip, int page)
 703{
 704	return rk_nfc_write_page_hwecc(chip, NULL, 1, page);
 705}
 706
 707static int rk_nfc_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_on,
 708				int page)
 709{
 710	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 711	struct rk_nfc *nfc = nand_get_controller_data(chip);
 712	struct mtd_info *mtd = nand_to_mtd(chip);
 713	struct nand_ecc_ctrl *ecc = &chip->ecc;
 714	int i, pages_per_blk;
 715
 716	pages_per_blk = mtd->erasesize / mtd->writesize;
 717	if ((chip->options & NAND_IS_BOOT_MEDIUM) &&
 718	    (page < (pages_per_blk * rknand->boot_blks)) &&
 719	    rknand->boot_ecc != ecc->strength) {
 720		/*
 721		 * There's currently no method to notify the MTD framework that
 722		 * a different ECC strength is in use for the boot blocks.
 723		 */
 724		return -EIO;
 725	}
 726
 727	nand_read_page_op(chip, page, 0, NULL, 0);
 728	rk_nfc_read_buf(nfc, nfc->page_buf, mtd->writesize + mtd->oobsize);
 729	for (i = 0; i < ecc->steps; i++) {
 730		/*
 731		 * The first four bytes of OOB are reserved for the
 732		 * boot ROM. In some debugging cases, such as with a read,
 733		 * erase and write back test, these 4 bytes also must be
 734		 * saved somewhere, otherwise this information will be
 735		 * lost during a write back.
 736		 */
 737		if (!i)
 738			memcpy(rk_nfc_buf_to_oob_ptr(chip, ecc->steps - 1),
 739			       rk_nfc_oob_ptr(chip, i),
 740			       NFC_SYS_DATA_SIZE);
 741		else
 742			memcpy(rk_nfc_buf_to_oob_ptr(chip, i - 1),
 743			       rk_nfc_oob_ptr(chip, i),
 744			       NFC_SYS_DATA_SIZE);
 745
 746		/* Copy ECC data from the NFC buffer. */
 747		memcpy(rk_nfc_buf_to_oob_ecc_ptr(chip, i),
 748		       rk_nfc_oob_ptr(chip, i) + NFC_SYS_DATA_SIZE,
 749		       ecc->bytes);
 750
 751		/* Copy data from the NFC buffer. */
 752		if (buf)
 753			memcpy(rk_nfc_buf_to_data_ptr(chip, buf, i),
 754			       rk_nfc_data_ptr(chip, i),
 755			       ecc->size);
 756	}
 757
 758	return 0;
 759}
 760
 761static int rk_nfc_read_page_hwecc(struct nand_chip *chip, u8 *buf, int oob_on,
 762				  int page)
 763{
 764	struct mtd_info *mtd = nand_to_mtd(chip);
 765	struct rk_nfc *nfc = nand_get_controller_data(chip);
 766	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 767	struct nand_ecc_ctrl *ecc = &chip->ecc;
 768	int oob_step = (ecc->bytes > 60) ? NFC_MAX_OOB_PER_STEP :
 769			NFC_MIN_OOB_PER_STEP;
 770	int pages_per_blk = mtd->erasesize / mtd->writesize;
 771	dma_addr_t dma_data, dma_oob;
 772	int ret = 0, i, cnt, boot_rom_mode = 0;
 773	int max_bitflips = 0, bch_st, ecc_fail = 0;
 774	u8 *oob;
 775	u32 tmp;
 776
 777	nand_read_page_op(chip, page, 0, NULL, 0);
 778
 779	dma_data = dma_map_single(nfc->dev, nfc->page_buf,
 780				  mtd->writesize,
 781				  DMA_FROM_DEVICE);
 782	if (dma_mapping_error(nfc->dev, dma_data))
 783		return -ENOMEM;
 784
 785	dma_oob = dma_map_single(nfc->dev, nfc->oob_buf,
 786				 ecc->steps * oob_step,
 787				 DMA_FROM_DEVICE);
 788	if (dma_mapping_error(nfc->dev, dma_oob)) {
 789		dma_unmap_single(nfc->dev, dma_data, mtd->writesize,
 790				 DMA_FROM_DEVICE);
 791		return -ENOMEM;
 792	}
 793
 794	/*
 795	 * The first blocks (4, 8 or 16 depending on the device)
 796	 * are used by the boot ROM.
 797	 * Configure the ECC algorithm supported by the boot ROM.
 798	 */
 799	if ((page < (pages_per_blk * rknand->boot_blks)) &&
 800	    (chip->options & NAND_IS_BOOT_MEDIUM)) {
 801		boot_rom_mode = 1;
 802		if (rknand->boot_ecc != ecc->strength)
 803			rk_nfc_hw_ecc_setup(chip, rknand->boot_ecc);
 804	}
 805
 806	reinit_completion(&nfc->done);
 807	writel(INT_DMA, nfc->regs + nfc->cfg->int_en_off);
 808	rk_nfc_xfer_start(nfc, NFC_READ, ecc->steps, dma_data,
 809			  dma_oob);
 810	ret = wait_for_completion_timeout(&nfc->done,
 811					  msecs_to_jiffies(100));
 812	if (!ret)
 813		dev_warn(nfc->dev, "read: wait dma done timeout.\n");
 814	/*
 815	 * Whether the DMA transfer is completed or not. The driver
 816	 * needs to check the NFC`s status register to see if the data
 817	 * transfer was completed.
 818	 */
 819	ret = rk_nfc_wait_for_xfer_done(nfc);
 820
 821	dma_unmap_single(nfc->dev, dma_data, mtd->writesize,
 822			 DMA_FROM_DEVICE);
 823	dma_unmap_single(nfc->dev, dma_oob, ecc->steps * oob_step,
 824			 DMA_FROM_DEVICE);
 825
 826	if (ret) {
 827		ret = -ETIMEDOUT;
 828		dev_err(nfc->dev, "read: wait transfer done timeout.\n");
 829		goto timeout_err;
 830	}
 831
 832	for (i = 0; i < ecc->steps; i++) {
 833		if (!i)
 834			oob = chip->oob_poi + (ecc->steps - 1) * NFC_SYS_DATA_SIZE;
 835		else
 836			oob = chip->oob_poi + (i - 1) * NFC_SYS_DATA_SIZE;
 837
 838		if (nfc->cfg->type == NFC_V9)
 839			tmp = nfc->oob_buf[i];
 840		else
 841			tmp = nfc->oob_buf[i * (oob_step / 4)];
 842
 843		*oob++ = (u8)tmp;
 844		*oob++ = (u8)(tmp >> 8);
 845		*oob++ = (u8)(tmp >> 16);
 846		*oob++ = (u8)(tmp >> 24);
 847	}
 848
 849	for (i = 0; i < (ecc->steps / 2); i++) {
 850		bch_st = readl_relaxed(nfc->regs +
 851				       nfc->cfg->bch_st_off + i * 4);
 852		if (bch_st & BIT(nfc->cfg->ecc0.err_flag_bit) ||
 853		    bch_st & BIT(nfc->cfg->ecc1.err_flag_bit)) {
 854			mtd->ecc_stats.failed++;
 855			ecc_fail = 1;
 856		} else {
 857			cnt = ECC_ERR_CNT(bch_st, nfc->cfg->ecc0);
 858			mtd->ecc_stats.corrected += cnt;
 859			max_bitflips = max_t(u32, max_bitflips, cnt);
 860
 861			cnt = ECC_ERR_CNT(bch_st, nfc->cfg->ecc1);
 862			mtd->ecc_stats.corrected += cnt;
 863			max_bitflips = max_t(u32, max_bitflips, cnt);
 864		}
 865	}
 866
 867	if (buf)
 868		memcpy(buf, nfc->page_buf, mtd->writesize);
 869
 870timeout_err:
 871	if (boot_rom_mode && rknand->boot_ecc != ecc->strength)
 872		rk_nfc_hw_ecc_setup(chip, ecc->strength);
 873
 874	if (ret)
 875		return ret;
 876
 877	if (ecc_fail) {
 878		dev_err(nfc->dev, "read page: %x ecc error!\n", page);
 879		return 0;
 880	}
 881
 882	return max_bitflips;
 883}
 884
 885static int rk_nfc_read_oob(struct nand_chip *chip, int page)
 886{
 887	return rk_nfc_read_page_hwecc(chip, NULL, 1, page);
 888}
 889
 890static inline void rk_nfc_hw_init(struct rk_nfc *nfc)
 891{
 892	/* Disable flash wp. */
 893	writel(FMCTL_WP, nfc->regs + NFC_FMCTL);
 894	/* Config default timing 40ns at 150 Mhz NFC clock. */
 895	writel(0x1081, nfc->regs + NFC_FMWAIT);
 896	nfc->cur_timing = 0x1081;
 897	/* Disable randomizer and DMA. */
 898	writel(0, nfc->regs + nfc->cfg->randmz_off);
 899	writel(0, nfc->regs + nfc->cfg->dma_cfg_off);
 900	writel(FLCTL_RST, nfc->regs + nfc->cfg->flctl_off);
 901}
 902
 903static irqreturn_t rk_nfc_irq(int irq, void *id)
 904{
 905	struct rk_nfc *nfc = id;
 906	u32 sta, ien;
 907
 908	sta = readl_relaxed(nfc->regs + nfc->cfg->int_st_off);
 909	ien = readl_relaxed(nfc->regs + nfc->cfg->int_en_off);
 910
 911	if (!(sta & ien))
 912		return IRQ_NONE;
 913
 914	writel(sta, nfc->regs + nfc->cfg->int_clr_off);
 915	writel(~sta & ien, nfc->regs + nfc->cfg->int_en_off);
 916
 917	complete(&nfc->done);
 918
 919	return IRQ_HANDLED;
 920}
 921
 922static int rk_nfc_enable_clks(struct device *dev, struct rk_nfc *nfc)
 923{
 924	int ret;
 925
 926	if (!IS_ERR(nfc->nfc_clk)) {
 927		ret = clk_prepare_enable(nfc->nfc_clk);
 928		if (ret) {
 929			dev_err(dev, "failed to enable NFC clk\n");
 930			return ret;
 931		}
 932	}
 933
 934	ret = clk_prepare_enable(nfc->ahb_clk);
 935	if (ret) {
 936		dev_err(dev, "failed to enable ahb clk\n");
 937		clk_disable_unprepare(nfc->nfc_clk);
 938		return ret;
 939	}
 940
 941	return 0;
 942}
 943
 944static void rk_nfc_disable_clks(struct rk_nfc *nfc)
 945{
 946	clk_disable_unprepare(nfc->nfc_clk);
 947	clk_disable_unprepare(nfc->ahb_clk);
 948}
 949
 950static int rk_nfc_ooblayout_free(struct mtd_info *mtd, int section,
 951				 struct mtd_oob_region *oob_region)
 952{
 953	struct nand_chip *chip = mtd_to_nand(mtd);
 954	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 955
 956	if (section)
 957		return -ERANGE;
 958
 959	oob_region->length = rknand->metadata_size - NFC_SYS_DATA_SIZE - 2;
 960	oob_region->offset = 2;
 961
 962	return 0;
 963}
 964
 965static int rk_nfc_ooblayout_ecc(struct mtd_info *mtd, int section,
 966				struct mtd_oob_region *oob_region)
 967{
 968	struct nand_chip *chip = mtd_to_nand(mtd);
 969	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 970
 971	if (section)
 972		return -ERANGE;
 973
 974	oob_region->length = mtd->oobsize - rknand->metadata_size;
 975	oob_region->offset = rknand->metadata_size;
 976
 977	return 0;
 978}
 979
 980static const struct mtd_ooblayout_ops rk_nfc_ooblayout_ops = {
 981	.free = rk_nfc_ooblayout_free,
 982	.ecc = rk_nfc_ooblayout_ecc,
 983};
 984
 985static int rk_nfc_ecc_init(struct device *dev, struct mtd_info *mtd)
 986{
 987	struct nand_chip *chip = mtd_to_nand(mtd);
 988	struct rk_nfc *nfc = nand_get_controller_data(chip);
 989	struct nand_ecc_ctrl *ecc = &chip->ecc;
 990	const u8 *strengths = nfc->cfg->ecc_strengths;
 991	u8 max_strength, nfc_max_strength;
 992	int i;
 993
 994	nfc_max_strength = nfc->cfg->ecc_strengths[0];
 995	/* If optional dt settings not present. */
 996	if (!ecc->size || !ecc->strength ||
 997	    ecc->strength > nfc_max_strength) {
 998		chip->ecc.size = 1024;
 999		ecc->steps = mtd->writesize / ecc->size;
1000
1001		/*
1002		 * HW ECC always requests the number of ECC bytes per 1024 byte
1003		 * blocks. The first 4 OOB bytes are reserved for sys data.
1004		 */
1005		max_strength = ((mtd->oobsize / ecc->steps) - 4) * 8 /
1006				 fls(8 * 1024);
1007		if (max_strength > nfc_max_strength)
1008			max_strength = nfc_max_strength;
1009
1010		for (i = 0; i < 4; i++) {
1011			if (max_strength >= strengths[i])
1012				break;
1013		}
1014
1015		if (i >= 4) {
1016			dev_err(nfc->dev, "unsupported ECC strength\n");
1017			return -EOPNOTSUPP;
1018		}
1019
1020		ecc->strength = strengths[i];
1021	}
1022	ecc->steps = mtd->writesize / ecc->size;
1023	ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * chip->ecc.size), 8);
1024
1025	return 0;
1026}
1027
1028static int rk_nfc_attach_chip(struct nand_chip *chip)
1029{
1030	struct mtd_info *mtd = nand_to_mtd(chip);
1031	struct device *dev = mtd->dev.parent;
1032	struct rk_nfc *nfc = nand_get_controller_data(chip);
1033	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
1034	struct nand_ecc_ctrl *ecc = &chip->ecc;
1035	int new_page_len, new_oob_len;
1036	void *buf;
1037	int ret;
1038
1039	if (chip->options & NAND_BUSWIDTH_16) {
1040		dev_err(dev, "16 bits bus width not supported");
1041		return -EINVAL;
1042	}
1043
1044	if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST)
1045		return 0;
1046
1047	ret = rk_nfc_ecc_init(dev, mtd);
1048	if (ret)
1049		return ret;
1050
1051	rknand->metadata_size = NFC_SYS_DATA_SIZE * ecc->steps;
1052
1053	if (rknand->metadata_size < NFC_SYS_DATA_SIZE + 2) {
1054		dev_err(dev,
1055			"driver needs at least %d bytes of meta data\n",
1056			NFC_SYS_DATA_SIZE + 2);
1057		return -EIO;
1058	}
1059
1060	/* Check buffer first, avoid duplicate alloc buffer. */
1061	new_page_len = mtd->writesize + mtd->oobsize;
1062	if (nfc->page_buf && new_page_len > nfc->page_buf_size) {
1063		buf = krealloc(nfc->page_buf, new_page_len,
1064			       GFP_KERNEL | GFP_DMA);
1065		if (!buf)
1066			return -ENOMEM;
1067		nfc->page_buf = buf;
1068		nfc->page_buf_size = new_page_len;
1069	}
1070
1071	new_oob_len = ecc->steps * NFC_MAX_OOB_PER_STEP;
1072	if (nfc->oob_buf && new_oob_len > nfc->oob_buf_size) {
1073		buf = krealloc(nfc->oob_buf, new_oob_len,
1074			       GFP_KERNEL | GFP_DMA);
1075		if (!buf) {
1076			kfree(nfc->page_buf);
1077			nfc->page_buf = NULL;
1078			return -ENOMEM;
1079		}
1080		nfc->oob_buf = buf;
1081		nfc->oob_buf_size = new_oob_len;
1082	}
1083
1084	if (!nfc->page_buf) {
1085		nfc->page_buf = kzalloc(new_page_len, GFP_KERNEL | GFP_DMA);
1086		if (!nfc->page_buf)
1087			return -ENOMEM;
1088		nfc->page_buf_size = new_page_len;
1089	}
1090
1091	if (!nfc->oob_buf) {
1092		nfc->oob_buf = kzalloc(new_oob_len, GFP_KERNEL | GFP_DMA);
1093		if (!nfc->oob_buf) {
1094			kfree(nfc->page_buf);
1095			nfc->page_buf = NULL;
1096			return -ENOMEM;
1097		}
1098		nfc->oob_buf_size = new_oob_len;
1099	}
1100
1101	chip->ecc.write_page_raw = rk_nfc_write_page_raw;
1102	chip->ecc.write_page = rk_nfc_write_page_hwecc;
1103	chip->ecc.write_oob = rk_nfc_write_oob;
1104
1105	chip->ecc.read_page_raw = rk_nfc_read_page_raw;
1106	chip->ecc.read_page = rk_nfc_read_page_hwecc;
1107	chip->ecc.read_oob = rk_nfc_read_oob;
1108
1109	return 0;
1110}
1111
1112static const struct nand_controller_ops rk_nfc_controller_ops = {
1113	.attach_chip = rk_nfc_attach_chip,
1114	.exec_op = rk_nfc_exec_op,
1115	.setup_interface = rk_nfc_setup_interface,
1116};
1117
1118static int rk_nfc_nand_chip_init(struct device *dev, struct rk_nfc *nfc,
1119				 struct device_node *np)
1120{
1121	struct rk_nfc_nand_chip *rknand;
1122	struct nand_chip *chip;
1123	struct mtd_info *mtd;
1124	int nsels;
1125	u32 tmp;
1126	int ret;
1127	int i;
1128
1129	if (!of_get_property(np, "reg", &nsels))
1130		return -ENODEV;
1131	nsels /= sizeof(u32);
1132	if (!nsels || nsels > NFC_MAX_NSELS) {
1133		dev_err(dev, "invalid reg property size %d\n", nsels);
1134		return -EINVAL;
1135	}
1136
1137	rknand = devm_kzalloc(dev, struct_size(rknand, sels, nsels),
1138			      GFP_KERNEL);
1139	if (!rknand)
1140		return -ENOMEM;
1141
1142	rknand->nsels = nsels;
1143	for (i = 0; i < nsels; i++) {
1144		ret = of_property_read_u32_index(np, "reg", i, &tmp);
1145		if (ret) {
1146			dev_err(dev, "reg property failure : %d\n", ret);
1147			return ret;
1148		}
1149
1150		if (tmp >= NFC_MAX_NSELS) {
1151			dev_err(dev, "invalid CS: %u\n", tmp);
1152			return -EINVAL;
1153		}
1154
1155		if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1156			dev_err(dev, "CS %u already assigned\n", tmp);
1157			return -EINVAL;
1158		}
1159
1160		rknand->sels[i] = tmp;
1161	}
1162
1163	chip = &rknand->chip;
1164	chip->controller = &nfc->controller;
1165
1166	nand_set_flash_node(chip, np);
1167
1168	nand_set_controller_data(chip, nfc);
1169
1170	chip->options |= NAND_USES_DMA | NAND_NO_SUBPAGE_WRITE;
1171	chip->bbt_options = NAND_BBT_USE_FLASH | NAND_BBT_NO_OOB;
1172
1173	/* Set default mode in case dt entry is missing. */
1174	chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1175
1176	mtd = nand_to_mtd(chip);
1177	mtd->owner = THIS_MODULE;
1178	mtd->dev.parent = dev;
1179
1180	if (!mtd->name) {
1181		dev_err(nfc->dev, "NAND label property is mandatory\n");
1182		return -EINVAL;
1183	}
1184
1185	mtd_set_ooblayout(mtd, &rk_nfc_ooblayout_ops);
1186	rk_nfc_hw_init(nfc);
1187	ret = nand_scan(chip, nsels);
1188	if (ret)
1189		return ret;
1190
1191	if (chip->options & NAND_IS_BOOT_MEDIUM) {
1192		ret = of_property_read_u32(np, "rockchip,boot-blks", &tmp);
1193		rknand->boot_blks = ret ? 0 : tmp;
1194
1195		ret = of_property_read_u32(np, "rockchip,boot-ecc-strength",
1196					   &tmp);
1197		rknand->boot_ecc = ret ? chip->ecc.strength : tmp;
1198	}
1199
1200	ret = mtd_device_register(mtd, NULL, 0);
1201	if (ret) {
1202		dev_err(dev, "MTD parse partition error\n");
1203		nand_cleanup(chip);
1204		return ret;
1205	}
1206
1207	list_add_tail(&rknand->node, &nfc->chips);
1208
1209	return 0;
1210}
1211
1212static void rk_nfc_chips_cleanup(struct rk_nfc *nfc)
1213{
1214	struct rk_nfc_nand_chip *rknand, *tmp;
1215	struct nand_chip *chip;
1216	int ret;
1217
1218	list_for_each_entry_safe(rknand, tmp, &nfc->chips, node) {
1219		chip = &rknand->chip;
1220		ret = mtd_device_unregister(nand_to_mtd(chip));
1221		WARN_ON(ret);
1222		nand_cleanup(chip);
1223		list_del(&rknand->node);
1224	}
1225}
1226
1227static int rk_nfc_nand_chips_init(struct device *dev, struct rk_nfc *nfc)
1228{
1229	struct device_node *np = dev->of_node;
1230	int nchips = of_get_child_count(np);
1231	int ret;
1232
1233	if (!nchips || nchips > NFC_MAX_NSELS) {
1234		dev_err(nfc->dev, "incorrect number of NAND chips (%d)\n",
1235			nchips);
1236		return -EINVAL;
1237	}
1238
1239	for_each_child_of_node_scoped(np, nand_np) {
1240		ret = rk_nfc_nand_chip_init(dev, nfc, nand_np);
1241		if (ret) {
1242			rk_nfc_chips_cleanup(nfc);
1243			return ret;
1244		}
1245	}
1246
1247	return 0;
1248}
1249
1250static struct nfc_cfg nfc_v6_cfg = {
1251		.type			= NFC_V6,
1252		.ecc_strengths		= {60, 40, 24, 16},
1253		.ecc_cfgs		= {
1254			0x00040011, 0x00040001, 0x00000011, 0x00000001,
1255		},
1256		.flctl_off		= 0x08,
1257		.bchctl_off		= 0x0C,
1258		.dma_cfg_off		= 0x10,
1259		.dma_data_buf_off	= 0x14,
1260		.dma_oob_buf_off	= 0x18,
1261		.dma_st_off		= 0x1C,
1262		.bch_st_off		= 0x20,
1263		.randmz_off		= 0x150,
1264		.int_en_off		= 0x16C,
1265		.int_clr_off		= 0x170,
1266		.int_st_off		= 0x174,
1267		.oob0_off		= 0x200,
1268		.oob1_off		= 0x230,
1269		.ecc0			= {
1270			.err_flag_bit	= 2,
1271			.low		= 3,
1272			.low_mask	= 0x1F,
1273			.low_bn		= 5,
1274			.high		= 27,
1275			.high_mask	= 0x1,
1276		},
1277		.ecc1			= {
1278			.err_flag_bit	= 15,
1279			.low		= 16,
1280			.low_mask	= 0x1F,
1281			.low_bn		= 5,
1282			.high		= 29,
1283			.high_mask	= 0x1,
1284		},
1285};
1286
1287static struct nfc_cfg nfc_v8_cfg = {
1288		.type			= NFC_V8,
1289		.ecc_strengths		= {16, 16, 16, 16},
1290		.ecc_cfgs		= {
1291			0x00000001, 0x00000001, 0x00000001, 0x00000001,
1292		},
1293		.flctl_off		= 0x08,
1294		.bchctl_off		= 0x0C,
1295		.dma_cfg_off		= 0x10,
1296		.dma_data_buf_off	= 0x14,
1297		.dma_oob_buf_off	= 0x18,
1298		.dma_st_off		= 0x1C,
1299		.bch_st_off		= 0x20,
1300		.randmz_off		= 0x150,
1301		.int_en_off		= 0x16C,
1302		.int_clr_off		= 0x170,
1303		.int_st_off		= 0x174,
1304		.oob0_off		= 0x200,
1305		.oob1_off		= 0x230,
1306		.ecc0			= {
1307			.err_flag_bit	= 2,
1308			.low		= 3,
1309			.low_mask	= 0x1F,
1310			.low_bn		= 5,
1311			.high		= 27,
1312			.high_mask	= 0x1,
1313		},
1314		.ecc1			= {
1315			.err_flag_bit	= 15,
1316			.low		= 16,
1317			.low_mask	= 0x1F,
1318			.low_bn		= 5,
1319			.high		= 29,
1320			.high_mask	= 0x1,
1321		},
1322};
1323
1324static struct nfc_cfg nfc_v9_cfg = {
1325		.type			= NFC_V9,
1326		.ecc_strengths		= {70, 60, 40, 16},
1327		.ecc_cfgs		= {
1328			0x00000001, 0x06000001, 0x04000001, 0x02000001,
1329		},
1330		.flctl_off		= 0x10,
1331		.bchctl_off		= 0x20,
1332		.dma_cfg_off		= 0x30,
1333		.dma_data_buf_off	= 0x34,
1334		.dma_oob_buf_off	= 0x38,
1335		.dma_st_off		= 0x3C,
1336		.bch_st_off		= 0x150,
1337		.randmz_off		= 0x208,
1338		.int_en_off		= 0x120,
1339		.int_clr_off		= 0x124,
1340		.int_st_off		= 0x128,
1341		.oob0_off		= 0x200,
1342		.oob1_off		= 0x204,
1343		.ecc0			= {
1344			.err_flag_bit	= 2,
1345			.low		= 3,
1346			.low_mask	= 0x7F,
1347			.low_bn		= 7,
1348			.high		= 0,
1349			.high_mask	= 0x0,
1350		},
1351		.ecc1			= {
1352			.err_flag_bit	= 18,
1353			.low		= 19,
1354			.low_mask	= 0x7F,
1355			.low_bn		= 7,
1356			.high		= 0,
1357			.high_mask	= 0x0,
1358		},
1359};
1360
1361static const struct of_device_id rk_nfc_id_table[] = {
1362	{
1363		.compatible = "rockchip,px30-nfc",
1364		.data = &nfc_v9_cfg
1365	},
1366	{
1367		.compatible = "rockchip,rk2928-nfc",
1368		.data = &nfc_v6_cfg
1369	},
1370	{
1371		.compatible = "rockchip,rv1108-nfc",
1372		.data = &nfc_v8_cfg
1373	},
1374	{ /* sentinel */ }
1375};
1376MODULE_DEVICE_TABLE(of, rk_nfc_id_table);
1377
1378static int rk_nfc_probe(struct platform_device *pdev)
1379{
1380	struct device *dev = &pdev->dev;
1381	struct rk_nfc *nfc;
1382	int ret, irq;
1383
1384	nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1385	if (!nfc)
1386		return -ENOMEM;
1387
1388	nand_controller_init(&nfc->controller);
1389	INIT_LIST_HEAD(&nfc->chips);
1390	nfc->controller.ops = &rk_nfc_controller_ops;
1391
1392	nfc->cfg = of_device_get_match_data(dev);
1393	nfc->dev = dev;
1394
1395	init_completion(&nfc->done);
1396
1397	nfc->regs = devm_platform_ioremap_resource(pdev, 0);
1398	if (IS_ERR(nfc->regs)) {
1399		ret = PTR_ERR(nfc->regs);
1400		goto release_nfc;
1401	}
1402
1403	nfc->nfc_clk = devm_clk_get(dev, "nfc");
1404	if (IS_ERR(nfc->nfc_clk)) {
1405		dev_dbg(dev, "no NFC clk\n");
1406		/* Some earlier models, such as rk3066, have no NFC clk. */
1407	}
1408
1409	nfc->ahb_clk = devm_clk_get(dev, "ahb");
1410	if (IS_ERR(nfc->ahb_clk)) {
1411		dev_err(dev, "no ahb clk\n");
1412		ret = PTR_ERR(nfc->ahb_clk);
1413		goto release_nfc;
1414	}
1415
1416	ret = rk_nfc_enable_clks(dev, nfc);
1417	if (ret)
1418		goto release_nfc;
1419
1420	irq = platform_get_irq(pdev, 0);
1421	if (irq < 0) {
1422		ret = -EINVAL;
1423		goto clk_disable;
1424	}
1425
1426	writel(0, nfc->regs + nfc->cfg->int_en_off);
1427	ret = devm_request_irq(dev, irq, rk_nfc_irq, 0x0, "rk-nand", nfc);
1428	if (ret) {
1429		dev_err(dev, "failed to request NFC irq\n");
1430		goto clk_disable;
1431	}
1432
1433	platform_set_drvdata(pdev, nfc);
1434
1435	ret = rk_nfc_nand_chips_init(dev, nfc);
1436	if (ret) {
1437		dev_err(dev, "failed to init NAND chips\n");
1438		goto clk_disable;
1439	}
1440	return 0;
1441
1442clk_disable:
1443	rk_nfc_disable_clks(nfc);
1444release_nfc:
1445	return ret;
1446}
1447
1448static void rk_nfc_remove(struct platform_device *pdev)
1449{
1450	struct rk_nfc *nfc = platform_get_drvdata(pdev);
1451
1452	kfree(nfc->page_buf);
1453	kfree(nfc->oob_buf);
1454	rk_nfc_chips_cleanup(nfc);
1455	rk_nfc_disable_clks(nfc);
1456}
1457
1458static int __maybe_unused rk_nfc_suspend(struct device *dev)
1459{
1460	struct rk_nfc *nfc = dev_get_drvdata(dev);
1461
1462	rk_nfc_disable_clks(nfc);
1463
1464	return 0;
1465}
1466
1467static int __maybe_unused rk_nfc_resume(struct device *dev)
1468{
1469	struct rk_nfc *nfc = dev_get_drvdata(dev);
1470	struct rk_nfc_nand_chip *rknand;
1471	struct nand_chip *chip;
1472	int ret;
1473	u32 i;
1474
1475	ret = rk_nfc_enable_clks(dev, nfc);
1476	if (ret)
1477		return ret;
1478
1479	/* Reset NAND chip if VCC was powered off. */
1480	list_for_each_entry(rknand, &nfc->chips, node) {
1481		chip = &rknand->chip;
1482		for (i = 0; i < rknand->nsels; i++)
1483			nand_reset(chip, i);
1484	}
1485
1486	return 0;
1487}
1488
1489static const struct dev_pm_ops rk_nfc_pm_ops = {
1490	SET_SYSTEM_SLEEP_PM_OPS(rk_nfc_suspend, rk_nfc_resume)
1491};
1492
1493static struct platform_driver rk_nfc_driver = {
1494	.probe = rk_nfc_probe,
1495	.remove = rk_nfc_remove,
1496	.driver = {
1497		.name = "rockchip-nfc",
1498		.of_match_table = rk_nfc_id_table,
1499		.pm = &rk_nfc_pm_ops,
1500	},
1501};
1502
1503module_platform_driver(rk_nfc_driver);
1504
1505MODULE_LICENSE("Dual MIT/GPL");
1506MODULE_AUTHOR("Yifeng Zhao <yifeng.zhao@rock-chips.com>");
1507MODULE_DESCRIPTION("Rockchip Nand Flash Controller Driver");