pmecc.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright 2017 ATMEL
   4 * Copyright 2017 Free Electrons
   5 *
   6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
   7 *
   8 * Derived from the atmel_nand.c driver which contained the following
   9 * copyrights:
  10 *
  11 *   Copyright 2003 Rick Bronson
  12 *
  13 *   Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8)
  14 *	Copyright 2001 Thomas Gleixner (gleixner@autronix.de)
  15 *
  16 *   Derived from drivers/mtd/spia.c (removed in v3.8)
  17 *	Copyright 2000 Steven J. Hill (sjhill@cotw.com)
  18 *
  19 *   Add Hardware ECC support for AT91SAM9260 / AT91SAM9263
  20 *	Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007
  21 *
  22 *   Derived from Das U-Boot source code
  23 *	(u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)
  24 *      Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
  25 *
  26 *   Add Programmable Multibit ECC support for various AT91 SoC
  27 *	Copyright 2012 ATMEL, Hong Xu
  28 *
  29 *   Add Nand Flash Controller support for SAMA5 SoC
  30 *	Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com)
  31 *
  32 * The PMECC is an hardware assisted BCH engine, which means part of the
  33 * ECC algorithm is left to the software. The hardware/software repartition
  34 * is explained in the "PMECC Controller Functional Description" chapter in
  35 * Atmel datasheets, and some of the functions in this file are directly
  36 * implementing the algorithms described in the "Software Implementation"
  37 * sub-section.
  38 *
  39 * TODO: it seems that the software BCH implementation in lib/bch.c is already
  40 * providing some of the logic we are implementing here. It would be smart
  41 * to expose the needed lib/bch.c helpers/functions and re-use them here.
  42 */
  43
  44#include <linux/genalloc.h>
  45#include <linux/iopoll.h>
  46#include <linux/module.h>
  47#include <linux/mtd/rawnand.h>
  48#include <linux/of_irq.h>
  49#include <linux/of_platform.h>
  50#include <linux/platform_device.h>
  51#include <linux/slab.h>
  52
  53#include "pmecc.h"
  54
  55/* Galois field dimension */
  56#define PMECC_GF_DIMENSION_13			13
  57#define PMECC_GF_DIMENSION_14			14
  58
  59/* Primitive Polynomial used by PMECC */
  60#define PMECC_GF_13_PRIMITIVE_POLY		0x201b
  61#define PMECC_GF_14_PRIMITIVE_POLY		0x4443
  62
  63#define PMECC_LOOKUP_TABLE_SIZE_512		0x2000
  64#define PMECC_LOOKUP_TABLE_SIZE_1024		0x4000
  65
  66/* Time out value for reading PMECC status register */
  67#define PMECC_MAX_TIMEOUT_MS			100
  68
  69/* PMECC Register Definitions */
  70#define ATMEL_PMECC_CFG				0x0
  71#define PMECC_CFG_BCH_STRENGTH(x)		(x)
  72#define PMECC_CFG_BCH_STRENGTH_MASK		GENMASK(2, 0)
  73#define PMECC_CFG_SECTOR512			(0 << 4)
  74#define PMECC_CFG_SECTOR1024			(1 << 4)
  75#define PMECC_CFG_NSECTORS(x)			((fls(x) - 1) << 8)
  76#define PMECC_CFG_READ_OP			(0 << 12)
  77#define PMECC_CFG_WRITE_OP			(1 << 12)
  78#define PMECC_CFG_SPARE_ENABLE			BIT(16)
  79#define PMECC_CFG_AUTO_ENABLE			BIT(20)
  80
  81#define ATMEL_PMECC_SAREA			0x4
  82#define ATMEL_PMECC_SADDR			0x8
  83#define ATMEL_PMECC_EADDR			0xc
  84
  85#define ATMEL_PMECC_CLK				0x10
  86#define PMECC_CLK_133MHZ			(2 << 0)
  87
  88#define ATMEL_PMECC_CTRL			0x14
  89#define PMECC_CTRL_RST				BIT(0)
  90#define PMECC_CTRL_DATA				BIT(1)
  91#define PMECC_CTRL_USER				BIT(2)
  92#define PMECC_CTRL_ENABLE			BIT(4)
  93#define PMECC_CTRL_DISABLE			BIT(5)
  94
  95#define ATMEL_PMECC_SR				0x18
  96#define PMECC_SR_BUSY				BIT(0)
  97#define PMECC_SR_ENABLE				BIT(4)
  98
  99#define ATMEL_PMECC_IER				0x1c
 100#define ATMEL_PMECC_IDR				0x20
 101#define ATMEL_PMECC_IMR				0x24
 102#define ATMEL_PMECC_ISR				0x28
 103#define PMECC_ERROR_INT				BIT(0)
 104
 105#define ATMEL_PMECC_ECC(sector, n)		\
 106	((((sector) + 1) * 0x40) + (n))
 107
 108#define ATMEL_PMECC_REM(sector, n)		\
 109	((((sector) + 1) * 0x40) + ((n) * 4) + 0x200)
 110
 111/* PMERRLOC Register Definitions */
 112#define ATMEL_PMERRLOC_ELCFG			0x0
 113#define PMERRLOC_ELCFG_SECTOR_512		(0 << 0)
 114#define PMERRLOC_ELCFG_SECTOR_1024		(1 << 0)
 115#define PMERRLOC_ELCFG_NUM_ERRORS(n)		((n) << 16)
 116
 117#define ATMEL_PMERRLOC_ELPRIM			0x4
 118#define ATMEL_PMERRLOC_ELEN			0x8
 119#define ATMEL_PMERRLOC_ELDIS			0xc
 120#define PMERRLOC_DISABLE			BIT(0)
 121
 122#define ATMEL_PMERRLOC_ELSR			0x10
 123#define PMERRLOC_ELSR_BUSY			BIT(0)
 124
 125#define ATMEL_PMERRLOC_ELIER			0x14
 126#define ATMEL_PMERRLOC_ELIDR			0x18
 127#define ATMEL_PMERRLOC_ELIMR			0x1c
 128#define ATMEL_PMERRLOC_ELISR			0x20
 129#define PMERRLOC_ERR_NUM_MASK			GENMASK(12, 8)
 130#define PMERRLOC_CALC_DONE			BIT(0)
 131
 132#define ATMEL_PMERRLOC_SIGMA(x)			(((x) * 0x4) + 0x28)
 133
 134#define ATMEL_PMERRLOC_EL(offs, x)		(((x) * 0x4) + (offs))
 135
 136struct atmel_pmecc_gf_tables {
 137	u16 *alpha_to;
 138	u16 *index_of;
 139};
 140
 141struct atmel_pmecc_caps {
 142	const int *strengths;
 143	int nstrengths;
 144	int el_offset;
 145	bool correct_erased_chunks;
 146	bool clk_ctrl;
 147};
 148
 149struct atmel_pmecc {
 150	struct device *dev;
 151	const struct atmel_pmecc_caps *caps;
 152
 153	struct {
 154		void __iomem *base;
 155		void __iomem *errloc;
 156	} regs;
 157
 158	struct mutex lock;
 159};
 160
 161struct atmel_pmecc_user_conf_cache {
 162	u32 cfg;
 163	u32 sarea;
 164	u32 saddr;
 165	u32 eaddr;
 166};
 167
 168struct atmel_pmecc_user {
 169	struct atmel_pmecc_user_conf_cache cache;
 170	struct atmel_pmecc *pmecc;
 171	const struct atmel_pmecc_gf_tables *gf_tables;
 172	int eccbytes;
 173	s16 *partial_syn;
 174	s16 *si;
 175	s16 *lmu;
 176	s16 *smu;
 177	s32 *mu;
 178	s32 *dmu;
 179	s32 *delta;
 180	u32 isr;
 181};
 182
 183static DEFINE_MUTEX(pmecc_gf_tables_lock);
 184static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512;
 185static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024;
 186
 187static inline int deg(unsigned int poly)
 188{
 189	/* polynomial degree is the most-significant bit index */
 190	return fls(poly) - 1;
 191}
 192
 193static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly,
 194				       struct atmel_pmecc_gf_tables *gf_tables)
 195{
 196	unsigned int i, x = 1;
 197	const unsigned int k = BIT(deg(poly));
 198	unsigned int nn = BIT(mm) - 1;
 199
 200	/* primitive polynomial must be of degree m */
 201	if (k != (1u << mm))
 202		return -EINVAL;
 203
 204	for (i = 0; i < nn; i++) {
 205		gf_tables->alpha_to[i] = x;
 206		gf_tables->index_of[x] = i;
 207		if (i && (x == 1))
 208			/* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */
 209			return -EINVAL;
 210		x <<= 1;
 211		if (x & k)
 212			x ^= poly;
 213	}
 214	gf_tables->alpha_to[nn] = 1;
 215	gf_tables->index_of[0] = 0;
 216
 217	return 0;
 218}
 219
 220static const struct atmel_pmecc_gf_tables *
 221atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req)
 222{
 223	struct atmel_pmecc_gf_tables *gf_tables;
 224	unsigned int poly, degree, table_size;
 225	int ret;
 226
 227	if (req->ecc.sectorsize == 512) {
 228		degree = PMECC_GF_DIMENSION_13;
 229		poly = PMECC_GF_13_PRIMITIVE_POLY;
 230		table_size = PMECC_LOOKUP_TABLE_SIZE_512;
 231	} else {
 232		degree = PMECC_GF_DIMENSION_14;
 233		poly = PMECC_GF_14_PRIMITIVE_POLY;
 234		table_size = PMECC_LOOKUP_TABLE_SIZE_1024;
 235	}
 236
 237	gf_tables = kzalloc(sizeof(*gf_tables) +
 238			    (2 * table_size * sizeof(u16)),
 239			    GFP_KERNEL);
 240	if (!gf_tables)
 241		return ERR_PTR(-ENOMEM);
 242
 243	gf_tables->alpha_to = (void *)(gf_tables + 1);
 244	gf_tables->index_of = gf_tables->alpha_to + table_size;
 245
 246	ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables);
 247	if (ret) {
 248		kfree(gf_tables);
 249		return ERR_PTR(ret);
 250	}
 251
 252	return gf_tables;
 253}
 254
 255static const struct atmel_pmecc_gf_tables *
 256atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req)
 257{
 258	const struct atmel_pmecc_gf_tables **gf_tables, *ret;
 259
 260	mutex_lock(&pmecc_gf_tables_lock);
 261	if (req->ecc.sectorsize == 512)
 262		gf_tables = &pmecc_gf_tables_512;
 263	else
 264		gf_tables = &pmecc_gf_tables_1024;
 265
 266	ret = *gf_tables;
 267
 268	if (!ret) {
 269		ret = atmel_pmecc_create_gf_tables(req);
 270		if (!IS_ERR(ret))
 271			*gf_tables = ret;
 272	}
 273	mutex_unlock(&pmecc_gf_tables_lock);
 274
 275	return ret;
 276}
 277
 278static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc,
 279					struct atmel_pmecc_user_req *req)
 280{
 281	int i, max_eccbytes, eccbytes = 0, eccstrength = 0;
 282
 283	if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0)
 284		return -EINVAL;
 285
 286	if (req->ecc.ooboffset >= 0 &&
 287	    req->ecc.ooboffset + req->ecc.bytes > req->oobsize)
 288		return -EINVAL;
 289
 290	if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) {
 291		if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
 292			return -EINVAL;
 293
 294		if (req->pagesize > 512)
 295			req->ecc.sectorsize = 1024;
 296		else
 297			req->ecc.sectorsize = 512;
 298	}
 299
 300	if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024)
 301		return -EINVAL;
 302
 303	if (req->pagesize % req->ecc.sectorsize)
 304		return -EINVAL;
 305
 306	req->ecc.nsectors = req->pagesize / req->ecc.sectorsize;
 307
 308	max_eccbytes = req->ecc.bytes;
 309
 310	for (i = 0; i < pmecc->caps->nstrengths; i++) {
 311		int nbytes, strength = pmecc->caps->strengths[i];
 312
 313		if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH &&
 314		    strength < req->ecc.strength)
 315			continue;
 316
 317		nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize),
 318				      8);
 319		nbytes *= req->ecc.nsectors;
 320
 321		if (nbytes > max_eccbytes)
 322			break;
 323
 324		eccstrength = strength;
 325		eccbytes = nbytes;
 326
 327		if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
 328			break;
 329	}
 330
 331	if (!eccstrength)
 332		return -EINVAL;
 333
 334	req->ecc.bytes = eccbytes;
 335	req->ecc.strength = eccstrength;
 336
 337	if (req->ecc.ooboffset < 0)
 338		req->ecc.ooboffset = req->oobsize - eccbytes;
 339
 340	return 0;
 341}
 342
 343struct atmel_pmecc_user *
 344atmel_pmecc_create_user(struct atmel_pmecc *pmecc,
 345			struct atmel_pmecc_user_req *req)
 346{
 347	struct atmel_pmecc_user *user;
 348	const struct atmel_pmecc_gf_tables *gf_tables;
 349	int strength, size, ret;
 350
 351	ret = atmel_pmecc_prepare_user_req(pmecc, req);
 352	if (ret)
 353		return ERR_PTR(ret);
 354
 355	size = sizeof(*user);
 356	size = ALIGN(size, sizeof(u16));
 357	/* Reserve space for partial_syn, si and smu */
 358	size += ((2 * req->ecc.strength) + 1) * sizeof(u16) *
 359		(2 + req->ecc.strength + 2);
 360	/* Reserve space for lmu. */
 361	size += (req->ecc.strength + 1) * sizeof(u16);
 362	/* Reserve space for mu, dmu and delta. */
 363	size = ALIGN(size, sizeof(s32));
 364	size += (req->ecc.strength + 1) * sizeof(s32) * 3;
 365
 366	user = devm_kzalloc(pmecc->dev, size, GFP_KERNEL);
 367	if (!user)
 368		return ERR_PTR(-ENOMEM);
 369
 370	user->pmecc = pmecc;
 371
 372	user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16));
 373	user->si = user->partial_syn + ((2 * req->ecc.strength) + 1);
 374	user->lmu = user->si + ((2 * req->ecc.strength) + 1);
 375	user->smu = user->lmu + (req->ecc.strength + 1);
 376	user->mu = (s32 *)PTR_ALIGN(user->smu +
 377				    (((2 * req->ecc.strength) + 1) *
 378				     (req->ecc.strength + 2)),
 379				    sizeof(s32));
 380	user->dmu = user->mu + req->ecc.strength + 1;
 381	user->delta = user->dmu + req->ecc.strength + 1;
 382
 383	gf_tables = atmel_pmecc_get_gf_tables(req);
 384	if (IS_ERR(gf_tables))
 385		return ERR_CAST(gf_tables);
 386
 387	user->gf_tables = gf_tables;
 388
 389	user->eccbytes = req->ecc.bytes / req->ecc.nsectors;
 390
 391	for (strength = 0; strength < pmecc->caps->nstrengths; strength++) {
 392		if (pmecc->caps->strengths[strength] == req->ecc.strength)
 393			break;
 394	}
 395
 396	user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) |
 397			  PMECC_CFG_NSECTORS(req->ecc.nsectors);
 398
 399	if (req->ecc.sectorsize == 1024)
 400		user->cache.cfg |= PMECC_CFG_SECTOR1024;
 401
 402	user->cache.sarea = req->oobsize - 1;
 403	user->cache.saddr = req->ecc.ooboffset;
 404	user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1;
 405
 406	return user;
 407}
 408EXPORT_SYMBOL_GPL(atmel_pmecc_create_user);
 409
 410static int get_strength(struct atmel_pmecc_user *user)
 411{
 412	const int *strengths = user->pmecc->caps->strengths;
 413
 414	return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK];
 415}
 416
 417static int get_sectorsize(struct atmel_pmecc_user *user)
 418{
 419	return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512;
 420}
 421
 422static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector)
 423{
 424	int strength = get_strength(user);
 425	u32 value;
 426	int i;
 427
 428	/* Fill odd syndromes */
 429	for (i = 0; i < strength; i++) {
 430		value = readl_relaxed(user->pmecc->regs.base +
 431				      ATMEL_PMECC_REM(sector, i / 2));
 432		if (i & 1)
 433			value >>= 16;
 434
 435		user->partial_syn[(2 * i) + 1] = value;
 436	}
 437}
 438
 439static void atmel_pmecc_substitute(struct atmel_pmecc_user *user)
 440{
 441	int degree = get_sectorsize(user) == 512 ? 13 : 14;
 442	int cw_len = BIT(degree) - 1;
 443	int strength = get_strength(user);
 444	s16 *alpha_to = user->gf_tables->alpha_to;
 445	s16 *index_of = user->gf_tables->index_of;
 446	s16 *partial_syn = user->partial_syn;
 447	s16 *si;
 448	int i, j;
 449
 450	/*
 451	 * si[] is a table that holds the current syndrome value,
 452	 * an element of that table belongs to the field
 453	 */
 454	si = user->si;
 455
 456	memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1));
 457
 458	/* Computation 2t syndromes based on S(x) */
 459	/* Odd syndromes */
 460	for (i = 1; i < 2 * strength; i += 2) {
 461		for (j = 0; j < degree; j++) {
 462			if (partial_syn[i] & BIT(j))
 463				si[i] = alpha_to[i * j] ^ si[i];
 464		}
 465	}
 466	/* Even syndrome = (Odd syndrome) ** 2 */
 467	for (i = 2, j = 1; j <= strength; i = ++j << 1) {
 468		if (si[j] == 0) {
 469			si[i] = 0;
 470		} else {
 471			s16 tmp;
 472
 473			tmp = index_of[si[j]];
 474			tmp = (tmp * 2) % cw_len;
 475			si[i] = alpha_to[tmp];
 476		}
 477	}
 478}
 479
 480static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user)
 481{
 482	s16 *lmu = user->lmu;
 483	s16 *si = user->si;
 484	s32 *mu = user->mu;
 485	s32 *dmu = user->dmu;
 486	s32 *delta = user->delta;
 487	int degree = get_sectorsize(user) == 512 ? 13 : 14;
 488	int cw_len = BIT(degree) - 1;
 489	int strength = get_strength(user);
 490	int num = 2 * strength + 1;
 491	s16 *index_of = user->gf_tables->index_of;
 492	s16 *alpha_to = user->gf_tables->alpha_to;
 493	int i, j, k;
 494	u32 dmu_0_count, tmp;
 495	s16 *smu = user->smu;
 496
 497	/* index of largest delta */
 498	int ro;
 499	int largest;
 500	int diff;
 501
 502	dmu_0_count = 0;
 503
 504	/* First Row */
 505
 506	/* Mu */
 507	mu[0] = -1;
 508
 509	memset(smu, 0, sizeof(s16) * num);
 510	smu[0] = 1;
 511
 512	/* discrepancy set to 1 */
 513	dmu[0] = 1;
 514	/* polynom order set to 0 */
 515	lmu[0] = 0;
 516	delta[0] = (mu[0] * 2 - lmu[0]) >> 1;
 517
 518	/* Second Row */
 519
 520	/* Mu */
 521	mu[1] = 0;
 522	/* Sigma(x) set to 1 */
 523	memset(&smu[num], 0, sizeof(s16) * num);
 524	smu[num] = 1;
 525
 526	/* discrepancy set to S1 */
 527	dmu[1] = si[1];
 528
 529	/* polynom order set to 0 */
 530	lmu[1] = 0;
 531
 532	delta[1] = (mu[1] * 2 - lmu[1]) >> 1;
 533
 534	/* Init the Sigma(x) last row */
 535	memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num);
 536
 537	for (i = 1; i <= strength; i++) {
 538		mu[i + 1] = i << 1;
 539		/* Begin Computing Sigma (Mu+1) and L(mu) */
 540		/* check if discrepancy is set to 0 */
 541		if (dmu[i] == 0) {
 542			dmu_0_count++;
 543
 544			tmp = ((strength - (lmu[i] >> 1) - 1) / 2);
 545			if ((strength - (lmu[i] >> 1) - 1) & 0x1)
 546				tmp += 2;
 547			else
 548				tmp += 1;
 549
 550			if (dmu_0_count == tmp) {
 551				for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
 552					smu[(strength + 1) * num + j] =
 553							smu[i * num + j];
 554
 555				lmu[strength + 1] = lmu[i];
 556				return;
 557			}
 558
 559			/* copy polynom */
 560			for (j = 0; j <= lmu[i] >> 1; j++)
 561				smu[(i + 1) * num + j] = smu[i * num + j];
 562
 563			/* copy previous polynom order to the next */
 564			lmu[i + 1] = lmu[i];
 565		} else {
 566			ro = 0;
 567			largest = -1;
 568			/* find largest delta with dmu != 0 */
 569			for (j = 0; j < i; j++) {
 570				if ((dmu[j]) && (delta[j] > largest)) {
 571					largest = delta[j];
 572					ro = j;
 573				}
 574			}
 575
 576			/* compute difference */
 577			diff = (mu[i] - mu[ro]);
 578
 579			/* Compute degree of the new smu polynomial */
 580			if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
 581				lmu[i + 1] = lmu[i];
 582			else
 583				lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
 584
 585			/* Init smu[i+1] with 0 */
 586			for (k = 0; k < num; k++)
 587				smu[(i + 1) * num + k] = 0;
 588
 589			/* Compute smu[i+1] */
 590			for (k = 0; k <= lmu[ro] >> 1; k++) {
 591				s16 a, b, c;
 592
 593				if (!(smu[ro * num + k] && dmu[i]))
 594					continue;
 595
 596				a = index_of[dmu[i]];
 597				b = index_of[dmu[ro]];
 598				c = index_of[smu[ro * num + k]];
 599				tmp = a + (cw_len - b) + c;
 600				a = alpha_to[tmp % cw_len];
 601				smu[(i + 1) * num + (k + diff)] = a;
 602			}
 603
 604			for (k = 0; k <= lmu[i] >> 1; k++)
 605				smu[(i + 1) * num + k] ^= smu[i * num + k];
 606		}
 607
 608		/* End Computing Sigma (Mu+1) and L(mu) */
 609		/* In either case compute delta */
 610		delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
 611
 612		/* Do not compute discrepancy for the last iteration */
 613		if (i >= strength)
 614			continue;
 615
 616		for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
 617			tmp = 2 * (i - 1);
 618			if (k == 0) {
 619				dmu[i + 1] = si[tmp + 3];
 620			} else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
 621				s16 a, b, c;
 622
 623				a = index_of[smu[(i + 1) * num + k]];
 624				b = si[2 * (i - 1) + 3 - k];
 625				c = index_of[b];
 626				tmp = a + c;
 627				tmp %= cw_len;
 628				dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1];
 629			}
 630		}
 631	}
 632}
 633
 634static int atmel_pmecc_err_location(struct atmel_pmecc_user *user)
 635{
 636	int sector_size = get_sectorsize(user);
 637	int degree = sector_size == 512 ? 13 : 14;
 638	struct atmel_pmecc *pmecc = user->pmecc;
 639	int strength = get_strength(user);
 640	int ret, roots_nbr, i, err_nbr = 0;
 641	int num = (2 * strength) + 1;
 642	s16 *smu = user->smu;
 643	u32 val;
 644
 645	writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS);
 646
 647	for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) {
 648		writel_relaxed(smu[(strength + 1) * num + i],
 649			       pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i));
 650		err_nbr++;
 651	}
 652
 653	val = (err_nbr - 1) << 16;
 654	if (sector_size == 1024)
 655		val |= 1;
 656
 657	writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG);
 658	writel((sector_size * 8) + (degree * strength),
 659	       pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN);
 660
 661	ret = readl_relaxed_poll_timeout(pmecc->regs.errloc +
 662					 ATMEL_PMERRLOC_ELISR,
 663					 val, val & PMERRLOC_CALC_DONE, 0,
 664					 PMECC_MAX_TIMEOUT_MS * 1000);
 665	if (ret) {
 666		dev_err(pmecc->dev,
 667			"PMECC: Timeout to calculate error location.\n");
 668		return ret;
 669	}
 670
 671	roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8;
 672	/* Number of roots == degree of smu hence <= cap */
 673	if (roots_nbr == user->lmu[strength + 1] >> 1)
 674		return err_nbr - 1;
 675
 676	/*
 677	 * Number of roots does not match the degree of smu
 678	 * unable to correct error.
 679	 */
 680	return -EBADMSG;
 681}
 682
 683int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector,
 684			       void *data, void *ecc)
 685{
 686	struct atmel_pmecc *pmecc = user->pmecc;
 687	int sectorsize = get_sectorsize(user);
 688	int eccbytes = user->eccbytes;
 689	int i, nerrors;
 690
 691	if (!(user->isr & BIT(sector)))
 692		return 0;
 693
 694	atmel_pmecc_gen_syndrome(user, sector);
 695	atmel_pmecc_substitute(user);
 696	atmel_pmecc_get_sigma(user);
 697
 698	nerrors = atmel_pmecc_err_location(user);
 699	if (nerrors < 0)
 700		return nerrors;
 701
 702	for (i = 0; i < nerrors; i++) {
 703		const char *area;
 704		int byte, bit;
 705		u32 errpos;
 706		u8 *ptr;
 707
 708		errpos = readl_relaxed(pmecc->regs.errloc +
 709				ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i));
 710		errpos--;
 711
 712		byte = errpos / 8;
 713		bit = errpos % 8;
 714
 715		if (byte < sectorsize) {
 716			ptr = data + byte;
 717			area = "data";
 718		} else if (byte < sectorsize + eccbytes) {
 719			ptr = ecc + byte - sectorsize;
 720			area = "ECC";
 721		} else {
 722			dev_dbg(pmecc->dev,
 723				"Invalid errpos value (%d, max is %d)\n",
 724				errpos, (sectorsize + eccbytes) * 8);
 725			return -EINVAL;
 726		}
 727
 728		dev_dbg(pmecc->dev,
 729			"Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n",
 730			area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit)));
 731
 732		*ptr ^= BIT(bit);
 733	}
 734
 735	return nerrors;
 736}
 737EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector);
 738
 739bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user)
 740{
 741	return user->pmecc->caps->correct_erased_chunks;
 742}
 743EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks);
 744
 745void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user,
 746					int sector, void *ecc)
 747{
 748	struct atmel_pmecc *pmecc = user->pmecc;
 749	u8 *ptr = ecc;
 750	int i;
 751
 752	for (i = 0; i < user->eccbytes; i++)
 753		ptr[i] = readb_relaxed(pmecc->regs.base +
 754				       ATMEL_PMECC_ECC(sector, i));
 755}
 756EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes);
 757
 758void atmel_pmecc_reset(struct atmel_pmecc *pmecc)
 759{
 760	writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL);
 761	writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
 762}
 763EXPORT_SYMBOL_GPL(atmel_pmecc_reset);
 764
 765int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op)
 766{
 767	struct atmel_pmecc *pmecc = user->pmecc;
 768	u32 cfg;
 769
 770	if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) {
 771		dev_err(pmecc->dev, "Bad ECC operation!");
 772		return -EINVAL;
 773	}
 774
 775	mutex_lock(&user->pmecc->lock);
 776
 777	cfg = user->cache.cfg;
 778	if (op == NAND_ECC_WRITE)
 779		cfg |= PMECC_CFG_WRITE_OP;
 780	else
 781		cfg |= PMECC_CFG_AUTO_ENABLE;
 782
 783	writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG);
 784	writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA);
 785	writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR);
 786	writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR);
 787
 788	writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
 789	writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL);
 790
 791	return 0;
 792}
 793EXPORT_SYMBOL_GPL(atmel_pmecc_enable);
 794
 795void atmel_pmecc_disable(struct atmel_pmecc_user *user)
 796{
 797	atmel_pmecc_reset(user->pmecc);
 798	mutex_unlock(&user->pmecc->lock);
 799}
 800EXPORT_SYMBOL_GPL(atmel_pmecc_disable);
 801
 802int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user)
 803{
 804	struct atmel_pmecc *pmecc = user->pmecc;
 805	u32 status;
 806	int ret;
 807
 808	ret = readl_relaxed_poll_timeout(pmecc->regs.base +
 809					 ATMEL_PMECC_SR,
 810					 status, !(status & PMECC_SR_BUSY), 0,
 811					 PMECC_MAX_TIMEOUT_MS * 1000);
 812	if (ret) {
 813		dev_err(pmecc->dev,
 814			"Timeout while waiting for PMECC ready.\n");
 815		return ret;
 816	}
 817
 818	user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR);
 819
 820	return 0;
 821}
 822EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy);
 823
 824static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev,
 825					const struct atmel_pmecc_caps *caps,
 826					int pmecc_res_idx, int errloc_res_idx)
 827{
 828	struct device *dev = &pdev->dev;
 829	struct atmel_pmecc *pmecc;
 830
 831	pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL);
 832	if (!pmecc)
 833		return ERR_PTR(-ENOMEM);
 834
 835	pmecc->caps = caps;
 836	pmecc->dev = dev;
 837	mutex_init(&pmecc->lock);
 838
 839	pmecc->regs.base = devm_platform_ioremap_resource(pdev, pmecc_res_idx);
 840	if (IS_ERR(pmecc->regs.base))
 841		return ERR_CAST(pmecc->regs.base);
 842
 843	pmecc->regs.errloc = devm_platform_ioremap_resource(pdev, errloc_res_idx);
 844	if (IS_ERR(pmecc->regs.errloc))
 845		return ERR_CAST(pmecc->regs.errloc);
 846
 847	/* pmecc data setup time */
 848	if (caps->clk_ctrl)
 849		writel(PMECC_CLK_133MHZ, pmecc->regs.base + ATMEL_PMECC_CLK);
 850
 851	/* Disable all interrupts before registering the PMECC handler. */
 852	writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR);
 853	atmel_pmecc_reset(pmecc);
 854
 855	return pmecc;
 856}
 857
 858static void devm_atmel_pmecc_put(struct device *dev, void *res)
 859{
 860	struct atmel_pmecc **pmecc = res;
 861
 862	put_device((*pmecc)->dev);
 863}
 864
 865static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev,
 866						   struct device_node *np)
 867{
 868	struct platform_device *pdev;
 869	struct atmel_pmecc *pmecc, **ptr;
 870	int ret;
 871
 872	pdev = of_find_device_by_node(np);
 873	if (!pdev)
 874		return ERR_PTR(-EPROBE_DEFER);
 875	pmecc = platform_get_drvdata(pdev);
 876	if (!pmecc) {
 877		ret = -EPROBE_DEFER;
 878		goto err_put_device;
 879	}
 880
 881	ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL);
 882	if (!ptr) {
 883		ret = -ENOMEM;
 884		goto err_put_device;
 885	}
 886
 887	*ptr = pmecc;
 888
 889	devres_add(userdev, ptr);
 890
 891	return pmecc;
 892
 893err_put_device:
 894	put_device(&pdev->dev);
 895	return ERR_PTR(ret);
 896}
 897
 898static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 };
 899
 900static struct atmel_pmecc_caps at91sam9g45_caps = {
 901	.strengths = atmel_pmecc_strengths,
 902	.nstrengths = 5,
 903	.el_offset = 0x8c,
 904	.clk_ctrl = true,
 905};
 906
 907static struct atmel_pmecc_caps sama5d4_caps = {
 908	.strengths = atmel_pmecc_strengths,
 909	.nstrengths = 5,
 910	.el_offset = 0x8c,
 911	.correct_erased_chunks = true,
 912};
 913
 914static struct atmel_pmecc_caps sama5d2_caps = {
 915	.strengths = atmel_pmecc_strengths,
 916	.nstrengths = 6,
 917	.el_offset = 0xac,
 918	.correct_erased_chunks = true,
 919};
 920
 921static const struct of_device_id __maybe_unused atmel_pmecc_legacy_match[] = {
 922	{ .compatible = "atmel,sama5d4-nand", &sama5d4_caps },
 923	{ .compatible = "atmel,sama5d2-nand", &sama5d2_caps },
 924	{ /* sentinel */ }
 925};
 926
 927struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev)
 928{
 929	struct atmel_pmecc *pmecc;
 930	struct device_node *np;
 931
 932	if (!userdev)
 933		return ERR_PTR(-EINVAL);
 934
 935	if (!userdev->of_node)
 936		return NULL;
 937
 938	np = of_parse_phandle(userdev->of_node, "ecc-engine", 0);
 939	if (np) {
 940		pmecc = atmel_pmecc_get_by_node(userdev, np);
 941		of_node_put(np);
 942	} else {
 943		/*
 944		 * Support old DT bindings: in this case the PMECC iomem
 945		 * resources are directly defined in the user pdev at position
 946		 * 1 and 2. Extract all relevant information from there.
 947		 */
 948		struct platform_device *pdev = to_platform_device(userdev);
 949		const struct atmel_pmecc_caps *caps;
 950		const struct of_device_id *match;
 951
 952		/* No PMECC engine available. */
 953		if (!of_property_read_bool(userdev->of_node,
 954					   "atmel,has-pmecc"))
 955			return NULL;
 956
 957		caps = &at91sam9g45_caps;
 958
 959		/* Find the caps associated to the NAND dev node. */
 960		match = of_match_node(atmel_pmecc_legacy_match,
 961				      userdev->of_node);
 962		if (match && match->data)
 963			caps = match->data;
 964
 965		pmecc = atmel_pmecc_create(pdev, caps, 1, 2);
 966	}
 967
 968	return pmecc;
 969}
 970EXPORT_SYMBOL(devm_atmel_pmecc_get);
 971
 972static const struct of_device_id atmel_pmecc_match[] = {
 973	{ .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps },
 974	{ .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps },
 975	{ .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps },
 976	{ /* sentinel */ }
 977};
 978MODULE_DEVICE_TABLE(of, atmel_pmecc_match);
 979
 980static int atmel_pmecc_probe(struct platform_device *pdev)
 981{
 982	struct device *dev = &pdev->dev;
 983	const struct atmel_pmecc_caps *caps;
 984	struct atmel_pmecc *pmecc;
 985
 986	caps = of_device_get_match_data(&pdev->dev);
 987	if (!caps) {
 988		dev_err(dev, "Invalid caps\n");
 989		return -EINVAL;
 990	}
 991
 992	pmecc = atmel_pmecc_create(pdev, caps, 0, 1);
 993	if (IS_ERR(pmecc))
 994		return PTR_ERR(pmecc);
 995
 996	platform_set_drvdata(pdev, pmecc);
 997
 998	return 0;
 999}
1000
1001static struct platform_driver atmel_pmecc_driver = {
1002	.driver = {
1003		.name = "atmel-pmecc",
1004		.of_match_table = atmel_pmecc_match,
1005	},
1006	.probe = atmel_pmecc_probe,
1007};
1008module_platform_driver(atmel_pmecc_driver);
1009
1010MODULE_LICENSE("GPL");
1011MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>");
1012MODULE_DESCRIPTION("PMECC engine driver");