core.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2016-2017 Micron Technology, Inc.
   4 *
   5 * Authors:
   6 *	Peter Pan <peterpandong@micron.com>
   7 *	Boris Brezillon <boris.brezillon@bootlin.com>
   8 */
   9
  10#define pr_fmt(fmt)	"spi-nand: " fmt
  11
  12#include <linux/device.h>
  13#include <linux/jiffies.h>
  14#include <linux/kernel.h>
  15#include <linux/module.h>
  16#include <linux/mtd/spinand.h>
  17#include <linux/of.h>
  18#include <linux/slab.h>
  19#include <linux/string.h>
  20#include <linux/spi/spi.h>
  21#include <linux/spi/spi-mem.h>
  22
  23int spinand_read_reg_op(struct spinand_device *spinand, u8 reg, u8 *val)
  24{
  25	struct spi_mem_op op = SPINAND_GET_FEATURE_1S_1S_1S_OP(reg,
  26						      spinand->scratchbuf);
  27	int ret;
  28
  29	ret = spi_mem_exec_op(spinand->spimem, &op);
  30	if (ret)
  31		return ret;
  32
  33	*val = *spinand->scratchbuf;
  34	return 0;
  35}
  36
  37int spinand_write_reg_op(struct spinand_device *spinand, u8 reg, u8 val)
  38{
  39	struct spi_mem_op op = SPINAND_SET_FEATURE_1S_1S_1S_OP(reg,
  40						      spinand->scratchbuf);
  41
  42	*spinand->scratchbuf = val;
  43	return spi_mem_exec_op(spinand->spimem, &op);
  44}
  45
  46static int spinand_read_status(struct spinand_device *spinand, u8 *status)
  47{
  48	return spinand_read_reg_op(spinand, REG_STATUS, status);
  49}
  50
  51static int spinand_get_cfg(struct spinand_device *spinand, u8 *cfg)
  52{
  53	struct nand_device *nand = spinand_to_nand(spinand);
  54
  55	if (WARN_ON(spinand->cur_target < 0 ||
  56		    spinand->cur_target >= nand->memorg.ntargets))
  57		return -EINVAL;
  58
  59	*cfg = spinand->cfg_cache[spinand->cur_target];
  60	return 0;
  61}
  62
  63static int spinand_set_cfg(struct spinand_device *spinand, u8 cfg)
  64{
  65	struct nand_device *nand = spinand_to_nand(spinand);
  66	int ret;
  67
  68	if (WARN_ON(spinand->cur_target < 0 ||
  69		    spinand->cur_target >= nand->memorg.ntargets))
  70		return -EINVAL;
  71
  72	if (spinand->cfg_cache[spinand->cur_target] == cfg)
  73		return 0;
  74
  75	ret = spinand_write_reg_op(spinand, REG_CFG, cfg);
  76	if (ret)
  77		return ret;
  78
  79	spinand->cfg_cache[spinand->cur_target] = cfg;
  80	return 0;
  81}
  82
  83/**
  84 * spinand_upd_cfg() - Update the configuration register
  85 * @spinand: the spinand device
  86 * @mask: the mask encoding the bits to update in the config reg
  87 * @val: the new value to apply
  88 *
  89 * Update the configuration register.
  90 *
  91 * Return: 0 on success, a negative error code otherwise.
  92 */
  93int spinand_upd_cfg(struct spinand_device *spinand, u8 mask, u8 val)
  94{
  95	int ret;
  96	u8 cfg;
  97
  98	ret = spinand_get_cfg(spinand, &cfg);
  99	if (ret)
 100		return ret;
 101
 102	cfg &= ~mask;
 103	cfg |= val;
 104
 105	return spinand_set_cfg(spinand, cfg);
 106}
 107
 108/**
 109 * spinand_select_target() - Select a specific NAND target/die
 110 * @spinand: the spinand device
 111 * @target: the target/die to select
 112 *
 113 * Select a new target/die. If chip only has one die, this function is a NOOP.
 114 *
 115 * Return: 0 on success, a negative error code otherwise.
 116 */
 117int spinand_select_target(struct spinand_device *spinand, unsigned int target)
 118{
 119	struct nand_device *nand = spinand_to_nand(spinand);
 120	int ret;
 121
 122	if (WARN_ON(target >= nand->memorg.ntargets))
 123		return -EINVAL;
 124
 125	if (spinand->cur_target == target)
 126		return 0;
 127
 128	if (nand->memorg.ntargets == 1) {
 129		spinand->cur_target = target;
 130		return 0;
 131	}
 132
 133	ret = spinand->select_target(spinand, target);
 134	if (ret)
 135		return ret;
 136
 137	spinand->cur_target = target;
 138	return 0;
 139}
 140
 141static int spinand_read_cfg(struct spinand_device *spinand)
 142{
 143	struct nand_device *nand = spinand_to_nand(spinand);
 144	unsigned int target;
 145	int ret;
 146
 147	for (target = 0; target < nand->memorg.ntargets; target++) {
 148		ret = spinand_select_target(spinand, target);
 149		if (ret)
 150			return ret;
 151
 152		/*
 153		 * We use spinand_read_reg_op() instead of spinand_get_cfg()
 154		 * here to bypass the config cache.
 155		 */
 156		ret = spinand_read_reg_op(spinand, REG_CFG,
 157					  &spinand->cfg_cache[target]);
 158		if (ret)
 159			return ret;
 160	}
 161
 162	return 0;
 163}
 164
 165static int spinand_init_cfg_cache(struct spinand_device *spinand)
 166{
 167	struct nand_device *nand = spinand_to_nand(spinand);
 168	struct device *dev = &spinand->spimem->spi->dev;
 169
 170	spinand->cfg_cache = devm_kcalloc(dev,
 171					  nand->memorg.ntargets,
 172					  sizeof(*spinand->cfg_cache),
 173					  GFP_KERNEL);
 174	if (!spinand->cfg_cache)
 175		return -ENOMEM;
 176
 177	return 0;
 178}
 179
 180static int spinand_init_quad_enable(struct spinand_device *spinand)
 181{
 182	bool enable = false;
 183
 184	if (!(spinand->flags & SPINAND_HAS_QE_BIT))
 185		return 0;
 186
 187	if (spinand->op_templates.read_cache->data.buswidth == 4 ||
 188	    spinand->op_templates.write_cache->data.buswidth == 4 ||
 189	    spinand->op_templates.update_cache->data.buswidth == 4)
 190		enable = true;
 191
 192	return spinand_upd_cfg(spinand, CFG_QUAD_ENABLE,
 193			       enable ? CFG_QUAD_ENABLE : 0);
 194}
 195
 196static int spinand_ecc_enable(struct spinand_device *spinand,
 197			      bool enable)
 198{
 199	return spinand_upd_cfg(spinand, CFG_ECC_ENABLE,
 200			       enable ? CFG_ECC_ENABLE : 0);
 201}
 202
 203static int spinand_cont_read_enable(struct spinand_device *spinand,
 204				    bool enable)
 205{
 206	return spinand->set_cont_read(spinand, enable);
 207}
 208
 209static int spinand_check_ecc_status(struct spinand_device *spinand, u8 status)
 210{
 211	struct nand_device *nand = spinand_to_nand(spinand);
 212
 213	if (spinand->eccinfo.get_status)
 214		return spinand->eccinfo.get_status(spinand, status);
 215
 216	switch (status & STATUS_ECC_MASK) {
 217	case STATUS_ECC_NO_BITFLIPS:
 218		return 0;
 219
 220	case STATUS_ECC_HAS_BITFLIPS:
 221		/*
 222		 * We have no way to know exactly how many bitflips have been
 223		 * fixed, so let's return the maximum possible value so that
 224		 * wear-leveling layers move the data immediately.
 225		 */
 226		return nanddev_get_ecc_conf(nand)->strength;
 227
 228	case STATUS_ECC_UNCOR_ERROR:
 229		return -EBADMSG;
 230
 231	default:
 232		break;
 233	}
 234
 235	return -EINVAL;
 236}
 237
 238static int spinand_noecc_ooblayout_ecc(struct mtd_info *mtd, int section,
 239				       struct mtd_oob_region *region)
 240{
 241	return -ERANGE;
 242}
 243
 244static int spinand_noecc_ooblayout_free(struct mtd_info *mtd, int section,
 245					struct mtd_oob_region *region)
 246{
 247	if (section)
 248		return -ERANGE;
 249
 250	/* Reserve 2 bytes for the BBM. */
 251	region->offset = 2;
 252	region->length = 62;
 253
 254	return 0;
 255}
 256
 257static const struct mtd_ooblayout_ops spinand_noecc_ooblayout = {
 258	.ecc = spinand_noecc_ooblayout_ecc,
 259	.free = spinand_noecc_ooblayout_free,
 260};
 261
 262static int spinand_ondie_ecc_init_ctx(struct nand_device *nand)
 263{
 264	struct spinand_device *spinand = nand_to_spinand(nand);
 265	struct mtd_info *mtd = nanddev_to_mtd(nand);
 266	struct spinand_ondie_ecc_conf *engine_conf;
 267
 268	nand->ecc.ctx.conf.engine_type = NAND_ECC_ENGINE_TYPE_ON_DIE;
 269	nand->ecc.ctx.conf.step_size = nand->ecc.requirements.step_size;
 270	nand->ecc.ctx.conf.strength = nand->ecc.requirements.strength;
 271
 272	engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL);
 273	if (!engine_conf)
 274		return -ENOMEM;
 275
 276	nand->ecc.ctx.priv = engine_conf;
 277
 278	if (spinand->eccinfo.ooblayout)
 279		mtd_set_ooblayout(mtd, spinand->eccinfo.ooblayout);
 280	else
 281		mtd_set_ooblayout(mtd, &spinand_noecc_ooblayout);
 282
 283	return 0;
 284}
 285
 286static void spinand_ondie_ecc_cleanup_ctx(struct nand_device *nand)
 287{
 288	kfree(nand->ecc.ctx.priv);
 289}
 290
 291static int spinand_ondie_ecc_prepare_io_req(struct nand_device *nand,
 292					    struct nand_page_io_req *req)
 293{
 294	struct spinand_device *spinand = nand_to_spinand(nand);
 295	bool enable = (req->mode != MTD_OPS_RAW);
 296
 297	if (!enable && spinand->flags & SPINAND_NO_RAW_ACCESS)
 298		return -EOPNOTSUPP;
 299
 300	memset(spinand->oobbuf, 0xff, nanddev_per_page_oobsize(nand));
 301
 302	/* Only enable or disable the engine */
 303	return spinand_ecc_enable(spinand, enable);
 304}
 305
 306static int spinand_ondie_ecc_finish_io_req(struct nand_device *nand,
 307					   struct nand_page_io_req *req)
 308{
 309	struct spinand_ondie_ecc_conf *engine_conf = nand->ecc.ctx.priv;
 310	struct spinand_device *spinand = nand_to_spinand(nand);
 311	struct mtd_info *mtd = spinand_to_mtd(spinand);
 312	int ret;
 313
 314	if (req->mode == MTD_OPS_RAW)
 315		return 0;
 316
 317	/* Nothing to do when finishing a page write */
 318	if (req->type == NAND_PAGE_WRITE)
 319		return 0;
 320
 321	/* Finish a page read: check the status, report errors/bitflips */
 322	ret = spinand_check_ecc_status(spinand, engine_conf->status);
 323	if (ret == -EBADMSG) {
 324		mtd->ecc_stats.failed++;
 325	} else if (ret > 0) {
 326		unsigned int pages;
 327
 328		/*
 329		 * Continuous reads don't allow us to get the detail,
 330		 * so we may exagerate the actual number of corrected bitflips.
 331		 */
 332		if (!req->continuous)
 333			pages = 1;
 334		else
 335			pages = req->datalen / nanddev_page_size(nand);
 336
 337		mtd->ecc_stats.corrected += ret * pages;
 338	}
 339
 340	return ret;
 341}
 342
 343static const struct nand_ecc_engine_ops spinand_ondie_ecc_engine_ops = {
 344	.init_ctx = spinand_ondie_ecc_init_ctx,
 345	.cleanup_ctx = spinand_ondie_ecc_cleanup_ctx,
 346	.prepare_io_req = spinand_ondie_ecc_prepare_io_req,
 347	.finish_io_req = spinand_ondie_ecc_finish_io_req,
 348};
 349
 350static struct nand_ecc_engine spinand_ondie_ecc_engine = {
 351	.ops = &spinand_ondie_ecc_engine_ops,
 352};
 353
 354static void spinand_ondie_ecc_save_status(struct nand_device *nand, u8 status)
 355{
 356	struct spinand_ondie_ecc_conf *engine_conf = nand->ecc.ctx.priv;
 357
 358	if (nand->ecc.ctx.conf.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE &&
 359	    engine_conf)
 360		engine_conf->status = status;
 361}
 362
 363int spinand_write_enable_op(struct spinand_device *spinand)
 364{
 365	struct spi_mem_op op = SPINAND_WR_EN_DIS_1S_0_0_OP(true);
 366
 367	return spi_mem_exec_op(spinand->spimem, &op);
 368}
 369
 370static int spinand_load_page_op(struct spinand_device *spinand,
 371				const struct nand_page_io_req *req)
 372{
 373	struct nand_device *nand = spinand_to_nand(spinand);
 374	unsigned int row = nanddev_pos_to_row(nand, &req->pos);
 375	struct spi_mem_op op = SPINAND_PAGE_READ_1S_1S_0_OP(row);
 376
 377	return spi_mem_exec_op(spinand->spimem, &op);
 378}
 379
 380static int spinand_read_from_cache_op(struct spinand_device *spinand,
 381				      const struct nand_page_io_req *req)
 382{
 383	struct nand_device *nand = spinand_to_nand(spinand);
 384	struct mtd_info *mtd = spinand_to_mtd(spinand);
 385	struct spi_mem_dirmap_desc *rdesc;
 386	unsigned int nbytes = 0;
 387	void *buf = NULL;
 388	u16 column = 0;
 389	ssize_t ret;
 390
 391	if (req->datalen) {
 392		buf = spinand->databuf;
 393		if (!req->continuous)
 394			nbytes = nanddev_page_size(nand);
 395		else
 396			nbytes = round_up(req->dataoffs + req->datalen,
 397					  nanddev_page_size(nand));
 398		column = 0;
 399	}
 400
 401	if (req->ooblen) {
 402		nbytes += nanddev_per_page_oobsize(nand);
 403		if (!buf) {
 404			buf = spinand->oobbuf;
 405			column = nanddev_page_size(nand);
 406		}
 407	}
 408
 409	if (req->mode == MTD_OPS_RAW)
 410		rdesc = spinand->dirmaps[req->pos.plane].rdesc;
 411	else
 412		rdesc = spinand->dirmaps[req->pos.plane].rdesc_ecc;
 413
 414	if (spinand->flags & SPINAND_HAS_READ_PLANE_SELECT_BIT)
 415		column |= req->pos.plane << fls(nanddev_page_size(nand));
 416
 417	while (nbytes) {
 418		ret = spi_mem_dirmap_read(rdesc, column, nbytes, buf);
 419		if (ret < 0)
 420			return ret;
 421
 422		if (!ret || ret > nbytes)
 423			return -EIO;
 424
 425		nbytes -= ret;
 426		column += ret;
 427		buf += ret;
 428
 429		/*
 430		 * Dirmap accesses are allowed to toggle the CS.
 431		 * Toggling the CS during a continuous read is forbidden.
 432		 */
 433		if (nbytes && req->continuous) {
 434			/*
 435			 * Spi controller with broken support of continuous
 436			 * reading was detected. Disable future use of
 437			 * continuous reading and return -EAGAIN to retry
 438			 * reading within regular mode.
 439			 */
 440			spinand->cont_read_possible = false;
 441			return -EAGAIN;
 442		}
 443	}
 444
 445	if (req->datalen)
 446		memcpy(req->databuf.in, spinand->databuf + req->dataoffs,
 447		       req->datalen);
 448
 449	if (req->ooblen) {
 450		if (req->mode == MTD_OPS_AUTO_OOB)
 451			mtd_ooblayout_get_databytes(mtd, req->oobbuf.in,
 452						    spinand->oobbuf,
 453						    req->ooboffs,
 454						    req->ooblen);
 455		else
 456			memcpy(req->oobbuf.in, spinand->oobbuf + req->ooboffs,
 457			       req->ooblen);
 458	}
 459
 460	return 0;
 461}
 462
 463static int spinand_write_to_cache_op(struct spinand_device *spinand,
 464				     const struct nand_page_io_req *req)
 465{
 466	struct nand_device *nand = spinand_to_nand(spinand);
 467	struct mtd_info *mtd = spinand_to_mtd(spinand);
 468	struct spi_mem_dirmap_desc *wdesc;
 469	unsigned int nbytes, column = 0;
 470	void *buf = spinand->databuf;
 471	ssize_t ret;
 472
 473	/*
 474	 * Looks like PROGRAM LOAD (AKA write cache) does not necessarily reset
 475	 * the cache content to 0xFF (depends on vendor implementation), so we
 476	 * must fill the page cache entirely even if we only want to program
 477	 * the data portion of the page, otherwise we might corrupt the BBM or
 478	 * user data previously programmed in OOB area.
 479	 *
 480	 * Only reset the data buffer manually, the OOB buffer is prepared by
 481	 * ECC engines ->prepare_io_req() callback.
 482	 */
 483	nbytes = nanddev_page_size(nand) + nanddev_per_page_oobsize(nand);
 484	memset(spinand->databuf, 0xff, nanddev_page_size(nand));
 485
 486	if (req->datalen)
 487		memcpy(spinand->databuf + req->dataoffs, req->databuf.out,
 488		       req->datalen);
 489
 490	if (req->ooblen) {
 491		if (req->mode == MTD_OPS_AUTO_OOB)
 492			mtd_ooblayout_set_databytes(mtd, req->oobbuf.out,
 493						    spinand->oobbuf,
 494						    req->ooboffs,
 495						    req->ooblen);
 496		else
 497			memcpy(spinand->oobbuf + req->ooboffs, req->oobbuf.out,
 498			       req->ooblen);
 499	}
 500
 501	if (req->mode == MTD_OPS_RAW)
 502		wdesc = spinand->dirmaps[req->pos.plane].wdesc;
 503	else
 504		wdesc = spinand->dirmaps[req->pos.plane].wdesc_ecc;
 505
 506	if (spinand->flags & SPINAND_HAS_PROG_PLANE_SELECT_BIT)
 507		column |= req->pos.plane << fls(nanddev_page_size(nand));
 508
 509	while (nbytes) {
 510		ret = spi_mem_dirmap_write(wdesc, column, nbytes, buf);
 511		if (ret < 0)
 512			return ret;
 513
 514		if (!ret || ret > nbytes)
 515			return -EIO;
 516
 517		nbytes -= ret;
 518		column += ret;
 519		buf += ret;
 520	}
 521
 522	return 0;
 523}
 524
 525static int spinand_program_op(struct spinand_device *spinand,
 526			      const struct nand_page_io_req *req)
 527{
 528	struct nand_device *nand = spinand_to_nand(spinand);
 529	unsigned int row = nanddev_pos_to_row(nand, &req->pos);
 530	struct spi_mem_op op = SPINAND_PROG_EXEC_1S_1S_0_OP(row);
 531
 532	return spi_mem_exec_op(spinand->spimem, &op);
 533}
 534
 535static int spinand_erase_op(struct spinand_device *spinand,
 536			    const struct nand_pos *pos)
 537{
 538	struct nand_device *nand = spinand_to_nand(spinand);
 539	unsigned int row = nanddev_pos_to_row(nand, pos);
 540	struct spi_mem_op op = SPINAND_BLK_ERASE_1S_1S_0_OP(row);
 541
 542	return spi_mem_exec_op(spinand->spimem, &op);
 543}
 544
 545/**
 546 * spinand_wait() - Poll memory device status
 547 * @spinand: the spinand device
 548 * @initial_delay_us: delay in us before starting to poll
 549 * @poll_delay_us: time to sleep between reads in us
 550 * @s: the pointer to variable to store the value of REG_STATUS
 551 *
 552 * This function polls a status register (REG_STATUS) and returns when
 553 * the STATUS_READY bit is 0 or when the timeout has expired.
 554 *
 555 * Return: 0 on success, a negative error code otherwise.
 556 */
 557int spinand_wait(struct spinand_device *spinand, unsigned long initial_delay_us,
 558		 unsigned long poll_delay_us, u8 *s)
 559{
 560	struct spi_mem_op op = SPINAND_GET_FEATURE_1S_1S_1S_OP(REG_STATUS,
 561							       spinand->scratchbuf);
 562	u8 status;
 563	int ret;
 564
 565	ret = spi_mem_poll_status(spinand->spimem, &op, STATUS_BUSY, 0,
 566				  initial_delay_us,
 567				  poll_delay_us,
 568				  SPINAND_WAITRDY_TIMEOUT_MS);
 569	if (ret)
 570		return ret;
 571
 572	status = *spinand->scratchbuf;
 573	if (!(status & STATUS_BUSY))
 574		goto out;
 575
 576	/*
 577	 * Extra read, just in case the STATUS_READY bit has changed
 578	 * since our last check
 579	 */
 580	ret = spinand_read_status(spinand, &status);
 581	if (ret)
 582		return ret;
 583
 584out:
 585	if (s)
 586		*s = status;
 587
 588	return status & STATUS_BUSY ? -ETIMEDOUT : 0;
 589}
 590
 591static int spinand_read_id_op(struct spinand_device *spinand, u8 naddr,
 592			      u8 ndummy, u8 *buf)
 593{
 594	struct spi_mem_op op = SPINAND_READID_1S_1S_1S_OP(
 595		naddr, ndummy, spinand->scratchbuf, SPINAND_MAX_ID_LEN);
 596	int ret;
 597
 598	ret = spi_mem_exec_op(spinand->spimem, &op);
 599	if (!ret)
 600		memcpy(buf, spinand->scratchbuf, SPINAND_MAX_ID_LEN);
 601
 602	return ret;
 603}
 604
 605static int spinand_reset_op(struct spinand_device *spinand)
 606{
 607	struct spi_mem_op op = SPINAND_RESET_1S_0_0_OP;
 608	int ret;
 609
 610	ret = spi_mem_exec_op(spinand->spimem, &op);
 611	if (ret)
 612		return ret;
 613
 614	return spinand_wait(spinand,
 615			    SPINAND_RESET_INITIAL_DELAY_US,
 616			    SPINAND_RESET_POLL_DELAY_US,
 617			    NULL);
 618}
 619
 620static int spinand_lock_block(struct spinand_device *spinand, u8 lock)
 621{
 622	return spinand_write_reg_op(spinand, REG_BLOCK_LOCK, lock);
 623}
 624
 625/**
 626 * spinand_read_page() - Read a page
 627 * @spinand: the spinand device
 628 * @req: the I/O request
 629 *
 630 * Return: 0 or a positive number of bitflips corrected on success.
 631 * A negative error code otherwise.
 632 */
 633int spinand_read_page(struct spinand_device *spinand,
 634		      const struct nand_page_io_req *req)
 635{
 636	struct nand_device *nand = spinand_to_nand(spinand);
 637	u8 status;
 638	int ret;
 639
 640	ret = nand_ecc_prepare_io_req(nand, (struct nand_page_io_req *)req);
 641	if (ret)
 642		return ret;
 643
 644	ret = spinand_load_page_op(spinand, req);
 645	if (ret)
 646		return ret;
 647
 648	ret = spinand_wait(spinand,
 649			   SPINAND_READ_INITIAL_DELAY_US,
 650			   SPINAND_READ_POLL_DELAY_US,
 651			   &status);
 652	if (ret < 0)
 653		return ret;
 654
 655	spinand_ondie_ecc_save_status(nand, status);
 656
 657	ret = spinand_read_from_cache_op(spinand, req);
 658	if (ret)
 659		return ret;
 660
 661	return nand_ecc_finish_io_req(nand, (struct nand_page_io_req *)req);
 662}
 663
 664/**
 665 * spinand_write_page() - Write a page
 666 * @spinand: the spinand device
 667 * @req: the I/O request
 668 *
 669 * Return: 0 or a positive number of bitflips corrected on success.
 670 * A negative error code otherwise.
 671 */
 672int spinand_write_page(struct spinand_device *spinand,
 673		       const struct nand_page_io_req *req)
 674{
 675	struct nand_device *nand = spinand_to_nand(spinand);
 676	u8 status;
 677	int ret;
 678
 679	ret = nand_ecc_prepare_io_req(nand, (struct nand_page_io_req *)req);
 680	if (ret)
 681		return ret;
 682
 683	ret = spinand_write_enable_op(spinand);
 684	if (ret)
 685		return ret;
 686
 687	ret = spinand_write_to_cache_op(spinand, req);
 688	if (ret)
 689		return ret;
 690
 691	ret = spinand_program_op(spinand, req);
 692	if (ret)
 693		return ret;
 694
 695	ret = spinand_wait(spinand,
 696			   SPINAND_WRITE_INITIAL_DELAY_US,
 697			   SPINAND_WRITE_POLL_DELAY_US,
 698			   &status);
 699	if (ret)
 700		return ret;
 701
 702	if (status & STATUS_PROG_FAILED)
 703		return -EIO;
 704
 705	return nand_ecc_finish_io_req(nand, (struct nand_page_io_req *)req);
 706}
 707
 708static int spinand_mtd_regular_page_read(struct mtd_info *mtd, loff_t from,
 709					 struct mtd_oob_ops *ops,
 710					 unsigned int *max_bitflips)
 711{
 712	struct spinand_device *spinand = mtd_to_spinand(mtd);
 713	struct nand_device *nand = mtd_to_nanddev(mtd);
 714	struct mtd_ecc_stats old_stats;
 715	struct nand_io_iter iter;
 716	bool disable_ecc = false;
 717	bool ecc_failed = false;
 718	unsigned int retry_mode = 0;
 719	int ret;
 720
 721	old_stats = mtd->ecc_stats;
 722
 723	if (ops->mode == MTD_OPS_RAW || !mtd->ooblayout)
 724		disable_ecc = true;
 725
 726	nanddev_io_for_each_page(nand, NAND_PAGE_READ, from, ops, &iter) {
 727		if (disable_ecc)
 728			iter.req.mode = MTD_OPS_RAW;
 729
 730		ret = spinand_select_target(spinand, iter.req.pos.target);
 731		if (ret)
 732			break;
 733
 734read_retry:
 735		ret = spinand_read_page(spinand, &iter.req);
 736		if (ret < 0 && ret != -EBADMSG)
 737			break;
 738
 739		if (ret == -EBADMSG && spinand->set_read_retry) {
 740			if (spinand->read_retries && (++retry_mode <= spinand->read_retries)) {
 741				ret = spinand->set_read_retry(spinand, retry_mode);
 742				if (ret < 0) {
 743					spinand->set_read_retry(spinand, 0);
 744					return ret;
 745				}
 746
 747				/* Reset ecc_stats; retry */
 748				mtd->ecc_stats = old_stats;
 749				goto read_retry;
 750			} else {
 751				/* No more retry modes; real failure */
 752				ecc_failed = true;
 753			}
 754		} else if (ret == -EBADMSG) {
 755			ecc_failed = true;
 756		} else {
 757			*max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 758		}
 759
 760		ret = 0;
 761		ops->retlen += iter.req.datalen;
 762		ops->oobretlen += iter.req.ooblen;
 763
 764		/* Reset to retry mode 0 */
 765		if (retry_mode) {
 766			retry_mode = 0;
 767			ret = spinand->set_read_retry(spinand, retry_mode);
 768			if (ret < 0)
 769				return ret;
 770		}
 771	}
 772
 773	if (ecc_failed && !ret)
 774		ret = -EBADMSG;
 775
 776	return ret;
 777}
 778
 779static int spinand_mtd_continuous_page_read(struct mtd_info *mtd, loff_t from,
 780					    struct mtd_oob_ops *ops,
 781					    unsigned int *max_bitflips)
 782{
 783	struct spinand_device *spinand = mtd_to_spinand(mtd);
 784	struct nand_device *nand = mtd_to_nanddev(mtd);
 785	struct nand_io_iter iter;
 786	u8 status;
 787	int ret;
 788
 789	ret = spinand_cont_read_enable(spinand, true);
 790	if (ret)
 791		return ret;
 792
 793	/*
 794	 * The cache is divided into two halves. While one half of the cache has
 795	 * the requested data, the other half is loaded with the next chunk of data.
 796	 * Therefore, the host can read out the data continuously from page to page.
 797	 * Each data read must be a multiple of 4-bytes and full pages should be read;
 798	 * otherwise, the data output might get out of sequence from one read command
 799	 * to another.
 800	 */
 801	nanddev_io_for_each_block(nand, NAND_PAGE_READ, from, ops, &iter) {
 802		ret = spinand_select_target(spinand, iter.req.pos.target);
 803		if (ret)
 804			goto end_cont_read;
 805
 806		ret = nand_ecc_prepare_io_req(nand, &iter.req);
 807		if (ret)
 808			goto end_cont_read;
 809
 810		ret = spinand_load_page_op(spinand, &iter.req);
 811		if (ret)
 812			goto end_cont_read;
 813
 814		ret = spinand_wait(spinand, SPINAND_READ_INITIAL_DELAY_US,
 815				   SPINAND_READ_POLL_DELAY_US, NULL);
 816		if (ret < 0)
 817			goto end_cont_read;
 818
 819		ret = spinand_read_from_cache_op(spinand, &iter.req);
 820		if (ret)
 821			goto end_cont_read;
 822
 823		ops->retlen += iter.req.datalen;
 824
 825		ret = spinand_read_status(spinand, &status);
 826		if (ret)
 827			goto end_cont_read;
 828
 829		spinand_ondie_ecc_save_status(nand, status);
 830
 831		ret = nand_ecc_finish_io_req(nand, &iter.req);
 832		if (ret < 0)
 833			goto end_cont_read;
 834
 835		*max_bitflips = max_t(unsigned int, *max_bitflips, ret);
 836		ret = 0;
 837	}
 838
 839end_cont_read:
 840	/*
 841	 * Once all the data has been read out, the host can either pull CS#
 842	 * high and wait for tRST or manually clear the bit in the configuration
 843	 * register to terminate the continuous read operation. We have no
 844	 * guarantee the SPI controller drivers will effectively deassert the CS
 845	 * when we expect them to, so take the register based approach.
 846	 */
 847	spinand_cont_read_enable(spinand, false);
 848
 849	return ret;
 850}
 851
 852static void spinand_cont_read_init(struct spinand_device *spinand)
 853{
 854	struct nand_device *nand = spinand_to_nand(spinand);
 855	enum nand_ecc_engine_type engine_type = nand->ecc.ctx.conf.engine_type;
 856
 857	/* OOBs cannot be retrieved so external/on-host ECC engine won't work */
 858	if (spinand->set_cont_read &&
 859	    (engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE ||
 860	     engine_type == NAND_ECC_ENGINE_TYPE_NONE)) {
 861		spinand->cont_read_possible = true;
 862
 863		/*
 864		 * Ensure continuous read is disabled on probe.
 865		 * Some devices retain this state across soft reset,
 866		 * which leaves the OOB area inaccessible and results
 867		 * in false positive returns from spinand_isbad().
 868		 */
 869		spinand_cont_read_enable(spinand, false);
 870	}
 871}
 872
 873static bool spinand_use_cont_read(struct mtd_info *mtd, loff_t from,
 874				  struct mtd_oob_ops *ops)
 875{
 876	struct nand_device *nand = mtd_to_nanddev(mtd);
 877	struct spinand_device *spinand = nand_to_spinand(nand);
 878	struct nand_pos start_pos, end_pos;
 879
 880	if (!spinand->cont_read_possible)
 881		return false;
 882
 883	/* OOBs won't be retrieved */
 884	if (ops->ooblen || ops->oobbuf)
 885		return false;
 886
 887	nanddev_offs_to_pos(nand, from, &start_pos);
 888	nanddev_offs_to_pos(nand, from + ops->len - 1, &end_pos);
 889
 890	/*
 891	 * Continuous reads never cross LUN boundaries. Some devices don't
 892	 * support crossing planes boundaries. Some devices don't even support
 893	 * crossing blocks boundaries. The common case being to read through UBI,
 894	 * we will very rarely read two consequent blocks or more, so it is safer
 895	 * and easier (can be improved) to only enable continuous reads when
 896	 * reading within the same erase block.
 897	 */
 898	if (start_pos.target != end_pos.target ||
 899	    start_pos.plane != end_pos.plane ||
 900	    start_pos.eraseblock != end_pos.eraseblock)
 901		return false;
 902
 903	return start_pos.page < end_pos.page;
 904}
 905
 906static int spinand_mtd_read(struct mtd_info *mtd, loff_t from,
 907			    struct mtd_oob_ops *ops)
 908{
 909	struct spinand_device *spinand = mtd_to_spinand(mtd);
 910	struct mtd_ecc_stats old_stats;
 911	unsigned int max_bitflips = 0;
 912	int ret;
 913
 914	mutex_lock(&spinand->lock);
 915
 916	old_stats = mtd->ecc_stats;
 917
 918	if (spinand_use_cont_read(mtd, from, ops)) {
 919		ret = spinand_mtd_continuous_page_read(mtd, from, ops, &max_bitflips);
 920		if (ret == -EAGAIN && !spinand->cont_read_possible) {
 921			/*
 922			 * Spi controller with broken support of continuous
 923			 * reading was detected (see spinand_read_from_cache_op()),
 924			 * repeat reading in regular mode.
 925			 */
 926			ret = spinand_mtd_regular_page_read(mtd, from, ops, &max_bitflips);
 927		}
 928	} else {
 929		ret = spinand_mtd_regular_page_read(mtd, from, ops, &max_bitflips);
 930	}
 931
 932	if (ops->stats) {
 933		ops->stats->uncorrectable_errors +=
 934			mtd->ecc_stats.failed - old_stats.failed;
 935		ops->stats->corrected_bitflips +=
 936			mtd->ecc_stats.corrected - old_stats.corrected;
 937	}
 938
 939	mutex_unlock(&spinand->lock);
 940
 941	return ret ? ret : max_bitflips;
 942}
 943
 944static int spinand_mtd_write(struct mtd_info *mtd, loff_t to,
 945			     struct mtd_oob_ops *ops)
 946{
 947	struct spinand_device *spinand = mtd_to_spinand(mtd);
 948	struct nand_device *nand = mtd_to_nanddev(mtd);
 949	struct nand_io_iter iter;
 950	bool disable_ecc = false;
 951	int ret = 0;
 952
 953	if (ops->mode == MTD_OPS_RAW || !mtd->ooblayout)
 954		disable_ecc = true;
 955
 956	mutex_lock(&spinand->lock);
 957
 958	nanddev_io_for_each_page(nand, NAND_PAGE_WRITE, to, ops, &iter) {
 959		if (disable_ecc)
 960			iter.req.mode = MTD_OPS_RAW;
 961
 962		ret = spinand_select_target(spinand, iter.req.pos.target);
 963		if (ret)
 964			break;
 965
 966		ret = spinand_write_page(spinand, &iter.req);
 967		if (ret)
 968			break;
 969
 970		ops->retlen += iter.req.datalen;
 971		ops->oobretlen += iter.req.ooblen;
 972	}
 973
 974	mutex_unlock(&spinand->lock);
 975
 976	return ret;
 977}
 978
 979static bool spinand_isbad(struct nand_device *nand, const struct nand_pos *pos)
 980{
 981	struct spinand_device *spinand = nand_to_spinand(nand);
 982	u8 marker[2] = { };
 983	struct nand_page_io_req req = {
 984		.pos = *pos,
 985		.ooblen = sizeof(marker),
 986		.ooboffs = 0,
 987		.oobbuf.in = marker,
 988		.mode = MTD_OPS_RAW,
 989	};
 990	int ret;
 991
 992	spinand_select_target(spinand, pos->target);
 993
 994	ret = spinand_read_page(spinand, &req);
 995	if (ret == -EOPNOTSUPP) {
 996		/* Retry with ECC in case raw access is not supported */
 997		req.mode = MTD_OPS_PLACE_OOB;
 998		spinand_read_page(spinand, &req);
 999	}
1000
1001	if (marker[0] != 0xff || marker[1] != 0xff)
1002		return true;
1003
1004	return false;
1005}
1006
1007static int spinand_mtd_block_isbad(struct mtd_info *mtd, loff_t offs)
1008{
1009	struct nand_device *nand = mtd_to_nanddev(mtd);
1010	struct spinand_device *spinand = nand_to_spinand(nand);
1011	struct nand_pos pos;
1012	int ret;
1013
1014	nanddev_offs_to_pos(nand, offs, &pos);
1015	mutex_lock(&spinand->lock);
1016	ret = nanddev_isbad(nand, &pos);
1017	mutex_unlock(&spinand->lock);
1018
1019	return ret;
1020}
1021
1022static int spinand_markbad(struct nand_device *nand, const struct nand_pos *pos)
1023{
1024	struct spinand_device *spinand = nand_to_spinand(nand);
1025	u8 marker[2] = { };
1026	struct nand_page_io_req req = {
1027		.pos = *pos,
1028		.ooboffs = 0,
1029		.ooblen = sizeof(marker),
1030		.oobbuf.out = marker,
1031		.mode = MTD_OPS_RAW,
1032	};
1033	int ret;
1034
1035	ret = spinand_select_target(spinand, pos->target);
1036	if (ret)
1037		return ret;
1038
1039	ret = spinand_write_page(spinand, &req);
1040	if (ret == -EOPNOTSUPP) {
1041		/* Retry with ECC in case raw access is not supported */
1042		req.mode = MTD_OPS_PLACE_OOB;
1043		ret = spinand_write_page(spinand, &req);
1044	}
1045
1046	return ret;
1047}
1048
1049static int spinand_mtd_block_markbad(struct mtd_info *mtd, loff_t offs)
1050{
1051	struct nand_device *nand = mtd_to_nanddev(mtd);
1052	struct spinand_device *spinand = nand_to_spinand(nand);
1053	struct nand_pos pos;
1054	int ret;
1055
1056	nanddev_offs_to_pos(nand, offs, &pos);
1057	mutex_lock(&spinand->lock);
1058	ret = nanddev_markbad(nand, &pos);
1059	mutex_unlock(&spinand->lock);
1060
1061	return ret;
1062}
1063
1064static int spinand_erase(struct nand_device *nand, const struct nand_pos *pos)
1065{
1066	struct spinand_device *spinand = nand_to_spinand(nand);
1067	u8 status;
1068	int ret;
1069
1070	ret = spinand_select_target(spinand, pos->target);
1071	if (ret)
1072		return ret;
1073
1074	ret = spinand_write_enable_op(spinand);
1075	if (ret)
1076		return ret;
1077
1078	ret = spinand_erase_op(spinand, pos);
1079	if (ret)
1080		return ret;
1081
1082	ret = spinand_wait(spinand,
1083			   SPINAND_ERASE_INITIAL_DELAY_US,
1084			   SPINAND_ERASE_POLL_DELAY_US,
1085			   &status);
1086
1087	if (!ret && (status & STATUS_ERASE_FAILED))
1088		ret = -EIO;
1089
1090	return ret;
1091}
1092
1093static int spinand_mtd_erase(struct mtd_info *mtd,
1094			     struct erase_info *einfo)
1095{
1096	struct spinand_device *spinand = mtd_to_spinand(mtd);
1097	int ret;
1098
1099	mutex_lock(&spinand->lock);
1100	ret = nanddev_mtd_erase(mtd, einfo);
1101	mutex_unlock(&spinand->lock);
1102
1103	return ret;
1104}
1105
1106static int spinand_mtd_block_isreserved(struct mtd_info *mtd, loff_t offs)
1107{
1108	struct spinand_device *spinand = mtd_to_spinand(mtd);
1109	struct nand_device *nand = mtd_to_nanddev(mtd);
1110	struct nand_pos pos;
1111	int ret;
1112
1113	nanddev_offs_to_pos(nand, offs, &pos);
1114	mutex_lock(&spinand->lock);
1115	ret = nanddev_isreserved(nand, &pos);
1116	mutex_unlock(&spinand->lock);
1117
1118	return ret;
1119}
1120
1121static struct spi_mem_dirmap_desc *spinand_create_rdesc(
1122					struct spinand_device *spinand,
1123					struct spi_mem_dirmap_info *info)
1124{
1125	struct nand_device *nand = spinand_to_nand(spinand);
1126	struct spi_mem_dirmap_desc *desc = NULL;
1127
1128	if (spinand->cont_read_possible) {
1129		/*
1130		 * spi controller may return an error if info->length is
1131		 * too large
1132		 */
1133		info->length = nanddev_eraseblock_size(nand);
1134		desc = devm_spi_mem_dirmap_create(&spinand->spimem->spi->dev,
1135						  spinand->spimem, info);
1136	}
1137
1138	if (IS_ERR_OR_NULL(desc)) {
1139		/*
1140		 * continuous reading is not supported by flash or
1141		 * its spi controller, use regular reading
1142		 */
1143		spinand->cont_read_possible = false;
1144
1145		info->length = nanddev_page_size(nand) +
1146			       nanddev_per_page_oobsize(nand);
1147		desc = devm_spi_mem_dirmap_create(&spinand->spimem->spi->dev,
1148						  spinand->spimem, info);
1149	}
1150
1151	return desc;
1152}
1153
1154static int spinand_create_dirmap(struct spinand_device *spinand,
1155				 unsigned int plane)
1156{
1157	struct nand_device *nand = spinand_to_nand(spinand);
1158	struct spi_mem_dirmap_info info = { 0 };
1159	struct spi_mem_dirmap_desc *desc;
1160
1161	/* The plane number is passed in MSB just above the column address */
1162	info.offset = plane << fls(nand->memorg.pagesize);
1163
1164	info.length = nanddev_page_size(nand) + nanddev_per_page_oobsize(nand);
1165	info.op_tmpl = *spinand->op_templates.update_cache;
1166	desc = devm_spi_mem_dirmap_create(&spinand->spimem->spi->dev,
1167					  spinand->spimem, &info);
1168	if (IS_ERR(desc))
1169		return PTR_ERR(desc);
1170
1171	spinand->dirmaps[plane].wdesc = desc;
1172
1173	info.op_tmpl = *spinand->op_templates.read_cache;
1174	desc = spinand_create_rdesc(spinand, &info);
1175	if (IS_ERR(desc))
1176		return PTR_ERR(desc);
1177
1178	spinand->dirmaps[plane].rdesc = desc;
1179
1180	if (nand->ecc.engine->integration != NAND_ECC_ENGINE_INTEGRATION_PIPELINED) {
1181		spinand->dirmaps[plane].wdesc_ecc = spinand->dirmaps[plane].wdesc;
1182		spinand->dirmaps[plane].rdesc_ecc = spinand->dirmaps[plane].rdesc;
1183
1184		return 0;
1185	}
1186
1187	info.length = nanddev_page_size(nand) + nanddev_per_page_oobsize(nand);
1188	info.op_tmpl = *spinand->op_templates.update_cache;
1189	info.op_tmpl.data.ecc = true;
1190	desc = devm_spi_mem_dirmap_create(&spinand->spimem->spi->dev,
1191					  spinand->spimem, &info);
1192	if (IS_ERR(desc))
1193		return PTR_ERR(desc);
1194
1195	spinand->dirmaps[plane].wdesc_ecc = desc;
1196
1197	info.op_tmpl = *spinand->op_templates.read_cache;
1198	info.op_tmpl.data.ecc = true;
1199	desc = spinand_create_rdesc(spinand, &info);
1200	if (IS_ERR(desc))
1201		return PTR_ERR(desc);
1202
1203	spinand->dirmaps[plane].rdesc_ecc = desc;
1204
1205	return 0;
1206}
1207
1208static int spinand_create_dirmaps(struct spinand_device *spinand)
1209{
1210	struct nand_device *nand = spinand_to_nand(spinand);
1211	int i, ret;
1212
1213	spinand->dirmaps = devm_kzalloc(&spinand->spimem->spi->dev,
1214					sizeof(*spinand->dirmaps) *
1215					nand->memorg.planes_per_lun,
1216					GFP_KERNEL);
1217	if (!spinand->dirmaps)
1218		return -ENOMEM;
1219
1220	for (i = 0; i < nand->memorg.planes_per_lun; i++) {
1221		ret = spinand_create_dirmap(spinand, i);
1222		if (ret)
1223			return ret;
1224	}
1225
1226	return 0;
1227}
1228
1229static const struct nand_ops spinand_ops = {
1230	.erase = spinand_erase,
1231	.markbad = spinand_markbad,
1232	.isbad = spinand_isbad,
1233};
1234
1235static const struct spinand_manufacturer *spinand_manufacturers[] = {
1236	&alliancememory_spinand_manufacturer,
1237	&ato_spinand_manufacturer,
1238	&esmt_8c_spinand_manufacturer,
1239	&esmt_c8_spinand_manufacturer,
1240	&fmsh_spinand_manufacturer,
1241	&foresee_spinand_manufacturer,
1242	&gigadevice_spinand_manufacturer,
1243	&macronix_spinand_manufacturer,
1244	&micron_spinand_manufacturer,
1245	&paragon_spinand_manufacturer,
1246	&skyhigh_spinand_manufacturer,
1247	&toshiba_spinand_manufacturer,
1248	&winbond_spinand_manufacturer,
1249	&xtx_spinand_manufacturer,
1250};
1251
1252static int spinand_manufacturer_match(struct spinand_device *spinand,
1253				      enum spinand_readid_method rdid_method)
1254{
1255	u8 *id = spinand->id.data;
1256	unsigned int i;
1257	int ret;
1258
1259	for (i = 0; i < ARRAY_SIZE(spinand_manufacturers); i++) {
1260		const struct spinand_manufacturer *manufacturer =
1261			spinand_manufacturers[i];
1262
1263		if (id[0] != manufacturer->id)
1264			continue;
1265
1266		ret = spinand_match_and_init(spinand,
1267					     manufacturer->chips,
1268					     manufacturer->nchips,
1269					     rdid_method);
1270		if (ret < 0)
1271			continue;
1272
1273		spinand->manufacturer = manufacturer;
1274		return 0;
1275	}
1276	return -EOPNOTSUPP;
1277}
1278
1279static int spinand_id_detect(struct spinand_device *spinand)
1280{
1281	u8 *id = spinand->id.data;
1282	int ret;
1283
1284	ret = spinand_read_id_op(spinand, 0, 0, id);
1285	if (ret)
1286		return ret;
1287	ret = spinand_manufacturer_match(spinand, SPINAND_READID_METHOD_OPCODE);
1288	if (!ret)
1289		return 0;
1290
1291	ret = spinand_read_id_op(spinand, 1, 0, id);
1292	if (ret)
1293		return ret;
1294	ret = spinand_manufacturer_match(spinand,
1295					 SPINAND_READID_METHOD_OPCODE_ADDR);
1296	if (!ret)
1297		return 0;
1298
1299	ret = spinand_read_id_op(spinand, 0, 1, id);
1300	if (ret)
1301		return ret;
1302	ret = spinand_manufacturer_match(spinand,
1303					 SPINAND_READID_METHOD_OPCODE_DUMMY);
1304
1305	return ret;
1306}
1307
1308static int spinand_manufacturer_init(struct spinand_device *spinand)
1309{
1310	int ret;
1311
1312	if (spinand->manufacturer->ops->init) {
1313		ret = spinand->manufacturer->ops->init(spinand);
1314		if (ret)
1315			return ret;
1316	}
1317
1318	if (spinand->configure_chip) {
1319		ret = spinand->configure_chip(spinand);
1320		if (ret)
1321			return ret;
1322	}
1323
1324	return 0;
1325}
1326
1327static void spinand_manufacturer_cleanup(struct spinand_device *spinand)
1328{
1329	/* Release manufacturer private data */
1330	if (spinand->manufacturer->ops->cleanup)
1331		return spinand->manufacturer->ops->cleanup(spinand);
1332}
1333
1334static const struct spi_mem_op *
1335spinand_select_op_variant(struct spinand_device *spinand,
1336			  const struct spinand_op_variants *variants)
1337{
1338	struct nand_device *nand = spinand_to_nand(spinand);
1339	const struct spi_mem_op *best_variant = NULL;
1340	u64 best_op_duration_ns = ULLONG_MAX;
1341	unsigned int i;
1342
1343	for (i = 0; i < variants->nops; i++) {
1344		struct spi_mem_op op = variants->ops[i];
1345		u64 op_duration_ns = 0;
1346		unsigned int nbytes;
1347		int ret;
1348
1349		nbytes = nanddev_per_page_oobsize(nand) +
1350			 nanddev_page_size(nand);
1351
1352		while (nbytes) {
1353			op.data.nbytes = nbytes;
1354			ret = spi_mem_adjust_op_size(spinand->spimem, &op);
1355			if (ret)
1356				break;
1357
1358			spi_mem_adjust_op_freq(spinand->spimem, &op);
1359
1360			if (!spi_mem_supports_op(spinand->spimem, &op))
1361				break;
1362
1363			nbytes -= op.data.nbytes;
1364
1365			op_duration_ns += spi_mem_calc_op_duration(spinand->spimem, &op);
1366		}
1367
1368		if (!nbytes && op_duration_ns < best_op_duration_ns) {
1369			best_op_duration_ns = op_duration_ns;
1370			best_variant = &variants->ops[i];
1371		}
1372	}
1373
1374	return best_variant;
1375}
1376
1377/**
1378 * spinand_match_and_init() - Try to find a match between a device ID and an
1379 *			      entry in a spinand_info table
1380 * @spinand: SPI NAND object
1381 * @table: SPI NAND device description table
1382 * @table_size: size of the device description table
1383 * @rdid_method: read id method to match
1384 *
1385 * Match between a device ID retrieved through the READ_ID command and an
1386 * entry in the SPI NAND description table. If a match is found, the spinand
1387 * object will be initialized with information provided by the matching
1388 * spinand_info entry.
1389 *
1390 * Return: 0 on success, a negative error code otherwise.
1391 */
1392int spinand_match_and_init(struct spinand_device *spinand,
1393			   const struct spinand_info *table,
1394			   unsigned int table_size,
1395			   enum spinand_readid_method rdid_method)
1396{
1397	u8 *id = spinand->id.data;
1398	struct nand_device *nand = spinand_to_nand(spinand);
1399	unsigned int i;
1400
1401	for (i = 0; i < table_size; i++) {
1402		const struct spinand_info *info = &table[i];
1403		const struct spi_mem_op *op;
1404
1405		if (rdid_method != info->devid.method)
1406			continue;
1407
1408		if (memcmp(id + 1, info->devid.id, info->devid.len))
1409			continue;
1410
1411		nand->memorg = table[i].memorg;
1412		nanddev_set_ecc_requirements(nand, &table[i].eccreq);
1413		spinand->eccinfo = table[i].eccinfo;
1414		spinand->flags = table[i].flags;
1415		spinand->id.len = 1 + table[i].devid.len;
1416		spinand->select_target = table[i].select_target;
1417		spinand->configure_chip = table[i].configure_chip;
1418		spinand->set_cont_read = table[i].set_cont_read;
1419		spinand->fact_otp = &table[i].fact_otp;
1420		spinand->user_otp = &table[i].user_otp;
1421		spinand->read_retries = table[i].read_retries;
1422		spinand->set_read_retry = table[i].set_read_retry;
1423
1424		op = spinand_select_op_variant(spinand,
1425					       info->op_variants.read_cache);
1426		if (!op)
1427			return -ENOTSUPP;
1428
1429		spinand->op_templates.read_cache = op;
1430
1431		op = spinand_select_op_variant(spinand,
1432					       info->op_variants.write_cache);
1433		if (!op)
1434			return -ENOTSUPP;
1435
1436		spinand->op_templates.write_cache = op;
1437
1438		op = spinand_select_op_variant(spinand,
1439					       info->op_variants.update_cache);
1440		spinand->op_templates.update_cache = op;
1441
1442		return 0;
1443	}
1444
1445	return -ENOTSUPP;
1446}
1447
1448static int spinand_detect(struct spinand_device *spinand)
1449{
1450	struct device *dev = &spinand->spimem->spi->dev;
1451	struct nand_device *nand = spinand_to_nand(spinand);
1452	int ret;
1453
1454	ret = spinand_reset_op(spinand);
1455	if (ret)
1456		return ret;
1457
1458	ret = spinand_id_detect(spinand);
1459	if (ret) {
1460		dev_err(dev, "unknown raw ID %*phN\n", SPINAND_MAX_ID_LEN,
1461			spinand->id.data);
1462		return ret;
1463	}
1464
1465	if (nand->memorg.ntargets > 1 && !spinand->select_target) {
1466		dev_err(dev,
1467			"SPI NANDs with more than one die must implement ->select_target()\n");
1468		return -EINVAL;
1469	}
1470
1471	dev_info(&spinand->spimem->spi->dev,
1472		 "%s SPI NAND was found.\n", spinand->manufacturer->name);
1473	dev_info(&spinand->spimem->spi->dev,
1474		 "%llu MiB, block size: %zu KiB, page size: %zu, OOB size: %u\n",
1475		 nanddev_size(nand) >> 20, nanddev_eraseblock_size(nand) >> 10,
1476		 nanddev_page_size(nand), nanddev_per_page_oobsize(nand));
1477
1478	return 0;
1479}
1480
1481static int spinand_init_flash(struct spinand_device *spinand)
1482{
1483	struct device *dev = &spinand->spimem->spi->dev;
1484	struct nand_device *nand = spinand_to_nand(spinand);
1485	int ret, i;
1486
1487	ret = spinand_read_cfg(spinand);
1488	if (ret)
1489		return ret;
1490
1491	ret = spinand_init_quad_enable(spinand);
1492	if (ret)
1493		return ret;
1494
1495	ret = spinand_upd_cfg(spinand, CFG_OTP_ENABLE, 0);
1496	if (ret)
1497		return ret;
1498
1499	ret = spinand_manufacturer_init(spinand);
1500	if (ret) {
1501		dev_err(dev,
1502		"Failed to initialize the SPI NAND chip (err = %d)\n",
1503		ret);
1504		return ret;
1505	}
1506
1507	/* After power up, all blocks are locked, so unlock them here. */
1508	for (i = 0; i < nand->memorg.ntargets; i++) {
1509		ret = spinand_select_target(spinand, i);
1510		if (ret)
1511			break;
1512
1513		ret = spinand_lock_block(spinand, BL_ALL_UNLOCKED);
1514		if (ret)
1515			break;
1516	}
1517
1518	if (ret)
1519		spinand_manufacturer_cleanup(spinand);
1520
1521	return ret;
1522}
1523
1524static void spinand_mtd_resume(struct mtd_info *mtd)
1525{
1526	struct spinand_device *spinand = mtd_to_spinand(mtd);
1527	int ret;
1528
1529	ret = spinand_reset_op(spinand);
1530	if (ret)
1531		return;
1532
1533	ret = spinand_init_flash(spinand);
1534	if (ret)
1535		return;
1536
1537	spinand_ecc_enable(spinand, false);
1538}
1539
1540static int spinand_init(struct spinand_device *spinand)
1541{
1542	struct device *dev = &spinand->spimem->spi->dev;
1543	struct mtd_info *mtd = spinand_to_mtd(spinand);
1544	struct nand_device *nand = mtd_to_nanddev(mtd);
1545	int ret;
1546
1547	/*
1548	 * We need a scratch buffer because the spi_mem interface requires that
1549	 * buf passed in spi_mem_op->data.buf be DMA-able.
1550	 */
1551	spinand->scratchbuf = kzalloc(SPINAND_MAX_ID_LEN, GFP_KERNEL);
1552	if (!spinand->scratchbuf)
1553		return -ENOMEM;
1554
1555	ret = spinand_detect(spinand);
1556	if (ret)
1557		goto err_free_bufs;
1558
1559	/*
1560	 * Use kzalloc() instead of devm_kzalloc() here, because some drivers
1561	 * may use this buffer for DMA access.
1562	 * Memory allocated by devm_ does not guarantee DMA-safe alignment.
1563	 */
1564	spinand->databuf = kzalloc(nanddev_eraseblock_size(nand),
1565				   GFP_KERNEL);
1566	if (!spinand->databuf) {
1567		ret = -ENOMEM;
1568		goto err_free_bufs;
1569	}
1570
1571	spinand->oobbuf = spinand->databuf + nanddev_page_size(nand);
1572
1573	ret = spinand_init_cfg_cache(spinand);
1574	if (ret)
1575		goto err_free_bufs;
1576
1577	ret = spinand_init_flash(spinand);
1578	if (ret)
1579		goto err_free_bufs;
1580
1581	ret = nanddev_init(nand, &spinand_ops, THIS_MODULE);
1582	if (ret)
1583		goto err_manuf_cleanup;
1584
1585	/* SPI-NAND default ECC engine is on-die */
1586	nand->ecc.defaults.engine_type = NAND_ECC_ENGINE_TYPE_ON_DIE;
1587	nand->ecc.ondie_engine = &spinand_ondie_ecc_engine;
1588
1589	spinand_ecc_enable(spinand, false);
1590	ret = nanddev_ecc_engine_init(nand);
1591	if (ret)
1592		goto err_cleanup_nanddev;
1593
1594	/*
1595	 * Continuous read can only be enabled with an on-die ECC engine, so the
1596	 * ECC initialization must have happened previously.
1597	 */
1598	spinand_cont_read_init(spinand);
1599
1600	mtd->_read_oob = spinand_mtd_read;
1601	mtd->_write_oob = spinand_mtd_write;
1602	mtd->_block_isbad = spinand_mtd_block_isbad;
1603	mtd->_block_markbad = spinand_mtd_block_markbad;
1604	mtd->_block_isreserved = spinand_mtd_block_isreserved;
1605	mtd->_erase = spinand_mtd_erase;
1606	mtd->_max_bad_blocks = nanddev_mtd_max_bad_blocks;
1607	mtd->_resume = spinand_mtd_resume;
1608
1609	if (spinand_user_otp_size(spinand) || spinand_fact_otp_size(spinand)) {
1610		ret = spinand_set_mtd_otp_ops(spinand);
1611		if (ret)
1612			goto err_cleanup_ecc_engine;
1613	}
1614
1615	if (nand->ecc.engine) {
1616		ret = mtd_ooblayout_count_freebytes(mtd);
1617		if (ret < 0)
1618			goto err_cleanup_ecc_engine;
1619	}
1620
1621	mtd->oobavail = ret;
1622
1623	/* Propagate ECC information to mtd_info */
1624	mtd->ecc_strength = nanddev_get_ecc_conf(nand)->strength;
1625	mtd->ecc_step_size = nanddev_get_ecc_conf(nand)->step_size;
1626	mtd->bitflip_threshold = DIV_ROUND_UP(mtd->ecc_strength * 3, 4);
1627
1628	ret = spinand_create_dirmaps(spinand);
1629	if (ret) {
1630		dev_err(dev,
1631			"Failed to create direct mappings for read/write operations (err = %d)\n",
1632			ret);
1633		goto err_cleanup_ecc_engine;
1634	}
1635
1636	return 0;
1637
1638err_cleanup_ecc_engine:
1639	nanddev_ecc_engine_cleanup(nand);
1640
1641err_cleanup_nanddev:
1642	nanddev_cleanup(nand);
1643
1644err_manuf_cleanup:
1645	spinand_manufacturer_cleanup(spinand);
1646
1647err_free_bufs:
1648	kfree(spinand->databuf);
1649	kfree(spinand->scratchbuf);
1650	return ret;
1651}
1652
1653static void spinand_cleanup(struct spinand_device *spinand)
1654{
1655	struct nand_device *nand = spinand_to_nand(spinand);
1656
1657	nanddev_ecc_engine_cleanup(nand);
1658	nanddev_cleanup(nand);
1659	spinand_manufacturer_cleanup(spinand);
1660	kfree(spinand->databuf);
1661	kfree(spinand->scratchbuf);
1662}
1663
1664static int spinand_probe(struct spi_mem *mem)
1665{
1666	struct spinand_device *spinand;
1667	struct mtd_info *mtd;
1668	int ret;
1669
1670	spinand = devm_kzalloc(&mem->spi->dev, sizeof(*spinand),
1671			       GFP_KERNEL);
1672	if (!spinand)
1673		return -ENOMEM;
1674
1675	spinand->spimem = mem;
1676	spi_mem_set_drvdata(mem, spinand);
1677	spinand_set_of_node(spinand, mem->spi->dev.of_node);
1678	mutex_init(&spinand->lock);
1679	mtd = spinand_to_mtd(spinand);
1680	mtd->dev.parent = &mem->spi->dev;
1681
1682	ret = spinand_init(spinand);
1683	if (ret)
1684		return ret;
1685
1686	ret = mtd_device_register(mtd, NULL, 0);
1687	if (ret)
1688		goto err_spinand_cleanup;
1689
1690	return 0;
1691
1692err_spinand_cleanup:
1693	spinand_cleanup(spinand);
1694
1695	return ret;
1696}
1697
1698static int spinand_remove(struct spi_mem *mem)
1699{
1700	struct spinand_device *spinand;
1701	struct mtd_info *mtd;
1702	int ret;
1703
1704	spinand = spi_mem_get_drvdata(mem);
1705	mtd = spinand_to_mtd(spinand);
1706
1707	ret = mtd_device_unregister(mtd);
1708	if (ret)
1709		return ret;
1710
1711	spinand_cleanup(spinand);
1712
1713	return 0;
1714}
1715
1716static const struct spi_device_id spinand_ids[] = {
1717	{ .name = "spi-nand" },
1718	{ /* sentinel */ },
1719};
1720MODULE_DEVICE_TABLE(spi, spinand_ids);
1721
1722#ifdef CONFIG_OF
1723static const struct of_device_id spinand_of_ids[] = {
1724	{ .compatible = "spi-nand" },
1725	{ /* sentinel */ },
1726};
1727MODULE_DEVICE_TABLE(of, spinand_of_ids);
1728#endif
1729
1730static struct spi_mem_driver spinand_drv = {
1731	.spidrv = {
1732		.id_table = spinand_ids,
1733		.driver = {
1734			.name = "spi-nand",
1735			.of_match_table = of_match_ptr(spinand_of_ids),
1736		},
1737	},
1738	.probe = spinand_probe,
1739	.remove = spinand_remove,
1740};
1741module_spi_mem_driver(spinand_drv);
1742
1743MODULE_DESCRIPTION("SPI NAND framework");
1744MODULE_AUTHOR("Peter Pan<peterpandong@micron.com>");
1745MODULE_LICENSE("GPL v2");