nand_cet.c

This is about correction error table in nand controller

/*
 NAND Correctable Error Table (cet) Support
---------------------------------------------
In case of a single bit correctable error, the block in which correctable error
occured is refreshed (i.e., read->erase->write the entire block). Following a
refresh a success value is returned by nand_read() i.e., the error is 
hidden from the file system. The Correctable Error Table (CET) keeps a history
(bit-vector) of per page correctable errors. If a correctable error happens 
on the same page twice, an error is returned to the file system.

The CET starts from the opposite end of BBT with 1-bit per page. The CET is 
initialized to all 1's. On the first correctable error the bit corresponding
to a page is reset. On an erase, all the bits of the corresponding block are
set. The CET can span across multiple blocks therefore a signature 'CET#' 
where # is the block number is kept in the OOB area of the first page of a 
CET block. Also, the total correctable error count is kept in the second
page OOB of the first CET block. 

There is an in-memory correctable error table during runtime which is flushed
to the flash every 10 mins (CET_SYNC_FREQ). 

 */

#include <linux/types.h>
#include <linux/mtd/mtd.h>
#include <linux/mtd/nand.h>
#include <linux/mtd/nand_ecc.h>
#include <linux/mtd/compatmac.h>
#include <linux/bitops.h>
#include <linux/vmalloc.h>
#include <linux/workqueue.h>
#include "nand_priv.h"

#ifdef CONFIG_MTD_NAND_CORRECTABLE_ERR_HANDLING

#define PRINTK(...)
#define BBT_PARTITION	(1<<20)
#define BBT_MAX_BLKS	4
#define CET_START_BLK(x, y) __ll_low(__ll_RightShift((x), (y)->bbt_erase_shift) - (BBT_PARTITION/(y)->blockSize));

#define CET_GOOD_BLK	0x00
#define CET_BAD_WEAR 	0x01
#define CET_BBT_USE	0x02
#define CET_BAD_FACTORY	0x03

#define CET_SYNC_FREQ	(10*60*HZ)	

static char cet_pattern[] = {'C', 'E', 'T', 0};
static struct nand_cet_descr cet_descr = {
	.offs = 9,
	.len = 4,
	.pattern = cet_pattern
};
static void sync_cet(void *);
static int search_cet_blks(struct mtd_info *, struct nand_cet_descr *, char);
extern char gClearCET;

/*
 * Private: Read OOB area in RAW mode
 */
static inline int nand_cet_read_oob(struct mtd_info *mtd, uint8_t *buf, loff_t offs)
{
	struct mtd_oob_ops ops;

	ops.mode = MTD_OOB_RAW;
	ops.len = mtd->oobsize;
	ops.ooblen = mtd->oobsize;
	ops.datbuf = NULL;
	ops.oobbuf = buf;
	ops.ooboffs = 0;

	return mtd->read_oob(mtd, offs, &ops);
}

/* 
 * Private: Write to the OOB area only
 */
static inline int nand_cet_write_oob(struct mtd_info *mtd, uint8_t *buf, loff_t offs)
{
	struct mtd_oob_ops ops;

	ops.mode = MTD_OOB_RAW;
	ops.len = mtd->oobsize;
	ops.ooblen = mtd->oobsize;
	ops.datbuf = NULL;
	ops.oobbuf = buf;
	ops.ooboffs = 0;

	return mtd->write_oob(mtd, offs, &ops);
}

/*
 * Private: write one page of data and OOB to flash 
 */
static int nand_cet_write(struct mtd_info *mtd, loff_t offs, size_t len, 
		uint8_t *buf, uint8_t *oob) 
{
	struct mtd_oob_ops ops;
	int ret;

	ops.mode = MTD_OOB_PLACE;
	ops.ooboffs = 0;
	ops.ooblen = mtd->oobsize;
	ops.datbuf = buf;
	ops.oobbuf = oob;
	ops.len = len;
	ret = mtd->write_oob(mtd, offs, &ops);

	return ret;
}

/*
 * bitcount - MIT Hackmem count implementation which is O(1)
 * http://infolab.stanford.edu/~manku/bitcount/bitcount.html
 * Counts the number of 1s in a given unsigned int n 
 */
static inline int bitcount(uint32_t n) 
{
	uint32_t tmp;
	tmp = n - ((n >> 1) & 033333333333)
		- ((n >> 2) & 011111111111);
	return ((tmp + (tmp >> 3)) & 030707070707) % 63;
}

/* 
 * Private debug function: Print OOBs 
 */
static void cet_printpg_oob(struct mtd_info *mtd, struct nand_cet_descr *cet, int count) 
{
	uint8_t oobbuf[mtd->oobsize];
	loff_t offs;
	int i, gdebug = 0;
	struct nand_chip *this = (struct nand_chip *) mtd->priv;

	offs = __ll_LeftShift32(cet->startblk, this->bbt_erase_shift);
	if (gdebug) {
		printk(KERN_INFO "%s: %x\n", __FUNCTION__, (unsigned int) offs);
	}
	for (i = 0; i < count; i++) {
		memset(oobbuf, 0, mtd->oobsize);
		if (nand_cet_read_oob(mtd, oobbuf, offs)) {
			return;
		}
		print_oobbuf((const char *) oobbuf, mtd->oobsize);
		offs = offs + cet->sign*this->pageSize;
	}
	return;
}

/*
 * Private debug function: Prints first OOB area of all blocks <block#, page0>
 */ 
static void cet_printblk_oob(struct mtd_info *mtd, struct nand_cet_descr *cet) 
{
	uint8_t *oobbuf;
	loff_t offs;
	int i;
	struct nand_chip *this = (struct nand_chip *) mtd->priv;

	if((oobbuf = (uint8_t *) vmalloc(sizeof(uint8_t)*mtd->oobsize)) == NULL) {
		printk(KERN_ERR "nandCET: %s vmalloc failed\n", __FUNCTION__);
		return;
	}
	for (i = 0; i < this->bbt_td->maxblocks; i++) {
		memset(oobbuf, 0, mtd->oobsize);
		offs = __ll_LeftShift32(cet->startblk+((cet->sign)*i), this->bbt_erase_shift);
		if (nand_cet_read_oob(mtd, oobbuf, offs)) {
			vfree(oobbuf);
			return;
		}
		print_oobbuf((const char *) oobbuf, mtd->oobsize);
	}
	vfree(oobbuf);
	return;
}

/*
 * Private debug function: erase all blocks belonging to CET 
 * Use for testing purposes only
 */
static void cet_eraseall(struct mtd_info *mtd, struct nand_cet_descr *cet) 
{
	int i, ret;
	loff_t from;
	struct erase_info einfo;
	struct nand_chip *this = (struct nand_chip *) mtd->priv;
	int gdebug = 0;

	for (i = 0; i < cet->numblks; i++) {
		if (cet->memtbl[i].blk != -1) {
			from = __ll_LeftShift32(cet->memtbl[i].blk, this->bbt_erase_shift);
			if (unlikely(gdebug)) {
				printk(KERN_INFO "DEBUG -> Erasing blk %x\n", cet->memtbl[i].blk);
			}
			memset(&einfo, 0, sizeof(einfo));
			einfo.mtd = mtd;
			einfo.addr = __ll_low(from);
			einfo.len = mtd->erasesize;
			ret = this->erase_bbt(mtd, &einfo, 1, 1);
			if (unlikely(ret < 0)) {
				printk(KERN_ERR "nandCET: %s Error erasing block %llx\n", __FUNCTION__, einfo.addr);
			}
		}
	}

	return;
}

/*
 * Private: Check if a block is factory marked bad block
 * Derived from nand_isbad_bbt()
 * Return values: 
 * 0x00 Good block
 * 0x01 Marked bad due to wear
 * 0x02 Reserved for BBT
 * 0x03 Factory marked bad
 */
static inline int check_badblk(struct mtd_info *mtd, loff_t offs) 
{
	struct nand_chip *this = (struct nand_chip *) mtd->priv;
	int blk;
	int res;

	blk = __ll_low(__ll_RightShift(offs, this->bbt_erase_shift-1));
	res = (this->bbt[blk >> 3] >> blk & 0x06) & 0x03;

	return res;
}

/*
 * Check for CET pattern in the OOB buffer
 * return the blk number present in the CET
 */
static inline int found_cet_pattern(struct nand_chip *this, uint8_t *buf)
{
	struct nand_cet_descr *cet = this->cet;
	int i;

	for (i = 0; i < cet->len-1; i++) {
		if (buf[cet->offs + i] != cet_pattern[i]) {
			return -1;
		}
	}
	return (int) buf[cet->offs + cet->len-1];
}

/*
 * Check for BBT/Mirror BBT pattern
 * Similar to the implementation in nand_bbt.c
 */
static inline int found_bbt_pattern(uint8_t *buf, struct nand_bbt_descr *bd) 
{
	int i;

	for (i = 0; i < bd->len; i++) {
		if (buf[bd->offs+i] != bd->pattern[i]) {
			return 0;
		}
	}
	return 1;
}

/*
 * Check OOB area to test if the block is erased 
 */
static inline int cet_iserased(struct mtd_info *mtd, uint8_t *oobbuf) 
{
	struct nand_chip *this = (struct nand_chip *) mtd->priv;
	struct nand_ecclayout *oobinfo = this->ecclayout;
	int i;

	for (i = 0; i < oobinfo->eccbytes; i++) {
		if (oobbuf[oobinfo->eccpos[i]] != 0xff) {
			return 0;
		}
	}
	return 1;
}

/*
 * Process kernel command line showcet
 * If the CET is loaded, display which blocks of flash the CET is in
 */
static inline void cmdline_showcet(struct mtd_info *mtd, struct nand_cet_descr *cet)
{
	int i;
	loff_t offs;
	uint8_t oobbuf[mtd->oobsize];
	struct nand_chip *this = (struct nand_chip *) mtd->priv;
	
	if (cet->flags == NAND_CET_DISABLED) {
		printk(KERN_INFO "nandCET: Disabled\n");
		return;
	}
	printk(KERN_INFO "nandCET: Correctable error count is 0x%x\n", cet->cerr_count);
	if (cet->flags == NAND_CET_LAZY) {
		printk(KERN_INFO "nandCET: Deferred until next correctable error\n");
		return;
	}
	printk(KERN_INFO "nandCET: Displaying first OOB area of all CET blocks ...\n");
	for (i = 0; i < cet->numblks; i++) {
		if (cet->memtbl[i].blk == -1) 
			continue;
		offs = __ll_LeftShift32(cet->memtbl[i].blk, this->bbt_erase_shift);
		printk(KERN_INFO "nandCET: Block[%d] @ %x\n", i, (unsigned int) offs);
		if (nand_cet_read_oob(mtd, oobbuf, offs)) {
			return;
		}
		print_oobbuf((const char *) oobbuf, mtd->oobsize);
	}
	return;
}

/*
 * Reset CET to all 0xffs 
 */
static inline int cmdline_resetcet(struct mtd_info *mtd, struct nand_cet_descr *cet)
{
	int i;

	cet_eraseall(mtd, cet);
	for (i = 0; i < cet->numblks; i++) {
		cet->memtbl[i].isdirty = 0;
		cet->memtbl[i].blk = -1;
		cet->memtbl[i].bitvec = NULL;
	}
	printk(KERN_INFO "nandCET: Recreating ... \n");

	return search_cet_blks(mtd, cet, 0);
}

/*
 * Create a CET pattern in the OOB area. 
 */
static int create_cet_blks(struct mtd_info *mtd, struct nand_cet_descr *cet)
{
	int i, j, ret, gdebug = 0;
	loff_t from;
	struct nand_bbt_descr *td, *md;
	struct erase_info einfo;
	struct nand_chip *this = (struct nand_chip *) mtd->priv;
	uint8_t oobbuf[mtd->oobsize];
	char *oobptr, count = 0;

	td = this->bbt_td;
	md = this->bbt_md;
	if (unlikely(gdebug)) {
		printk(KERN_INFO "nandCET: Inside %s\n", __FUNCTION__);
	}
	for (i = 0; i < td->maxblocks; i++) {
		from = __ll_LeftShift32(cet->startblk+i*cet->sign, this->bbt_erase_shift);
		/* Skip if bad block */
		ret = check_badblk(mtd, from);
		if (ret == CET_BAD_FACTORY || ret == CET_BAD_WEAR) {
			continue;
		}
		memset(oobbuf, 0, mtd->oobsize);
		if (nand_cet_read_oob(mtd, oobbuf, from)) {
			printk(KERN_INFO "nandCET: %s %d Error reading OOB\n", __FUNCTION__, __LINE__);
			return -1;
		}
		/* If BBT/MBT block found  we have no space left */
		if (found_bbt_pattern(oobbuf, td) || found_bbt_pattern(oobbuf, md)) {
			printk(KERN_INFO "nandCET: %s blk %x is BBT\n", __FUNCTION__, cet->startblk + i*cet->sign);
			return -1;
		}
		//if (!cet_iserased(mtd, oobbuf)) {
		if (unlikely(gdebug)) {
			printk(KERN_INFO "nandCET: block %x is erased\n", cet->startblk+i*cet->sign);
		}
		/* Erase */
		memset(&einfo, 0, sizeof(einfo));
		einfo.mtd = mtd;
		einfo.addr = __ll_low(from);
		einfo.len = mtd->erasesize;
		ret = this->erase_bbt(mtd, &einfo, 1, 1);
		if (unlikely(ret < 0)) {
			printk(KERN_ERR "nandCET: %s Error erasing block %x\n", __FUNCTION__, cet->startblk+i*cet->sign);
			return -1;
		}
		//} 
		/* Write 'CET#' pattern to the OOB area */
		memset(oobbuf, 0xff, mtd->oobsize);
		if (unlikely(gdebug)) {
			printk(KERN_INFO "nandCET: writing CET %d to OOB area\n", (int)count);
		}
		oobptr = (char *) oobbuf;
		for (j = 0; j < cet->len-1; j++) {
			oobptr[cet->offs + j] = cet->pattern[j];
		}
		oobptr[cet->offs + j] = count;
		if (nand_cet_write_oob(mtd, oobbuf, from)) {
			printk(KERN_ERR "nandCET: %s Error writing to OOB# %x\n", __FUNCTION__, (unsigned int)from);
			return -1;
		}
		/* If this is the first CET block, init the correctable erase count to 0 */
		if (count == 0) {
			memset(oobbuf, 0xff, mtd->oobsize);
			oobptr = (char *) oobbuf;
			*((uint32_t *) (oobptr + cet->offs)) = 0x00000000;
			from += this->pageSize;
			if (unlikely(gdebug)) {
				printk(KERN_INFO "DEBUG -> 0: from = %x\n", (unsigned int) from);
				printk(KERN_INFO "nandCET: Writing cer_count to page %x\n", (unsigned int) from);
			}
			if (nand_cet_write_oob(mtd, oobbuf, from)) {
				printk(KERN_INFO "nandCET: %s Error writing to OOB# %x\n", __FUNCTION__, (unsigned int)from);
				return -1;
			}
		}
		count++;
		if (((int)count) == cet->numblks) {
			return 0;
		}
	}
	return -1;
}

/*
 * Search for CET blocks
 * force => 1 Force creation of tables, do not defer for later
 */
static int search_cet_blks(struct mtd_info *mtd, struct nand_cet_descr *cet, char force)
{
	int i, count = 0, ret;
	loff_t from;
	struct nand_bbt_descr *td, *md;
	uint8_t oobbuf[mtd->oobsize];
	struct nand_chip *this = (struct nand_chip *) mtd->priv;
	int gdebug = 0;

	td = this->bbt_td;
	md = this->bbt_md;
	if (unlikely(gdebug)) {
		printk(KERN_INFO "DEBUG -> Inside search_cet_blks\n");
	}
	for (i = 0; i < td->maxblocks; i++) {
		from = __ll_LeftShift32(cet->startblk+i*cet->sign, this->bbt_erase_shift);
		/* Skip if bad block */
		ret = check_badblk(mtd, from);
		if (ret == CET_BAD_FACTORY || ret == CET_BAD_WEAR) {
			continue;
		}
		/* Read the OOB area of the first page of the block */
		memset(oobbuf, 0, mtd->oobsize);
		if (nand_cet_read_oob(mtd, oobbuf, from)) {
			printk(KERN_INFO "nandCET: %s %d Error reading OOB\n", __FUNCTION__, __LINE__);
			cet->flags = NAND_CET_DISABLED;
			return -1;
		}
		if (unlikely(gdebug)) {
			print_oobbuf(oobbuf, mtd->oobsize);
		}
		/* Return -1 if BBT/MBT block => no space left for CET */
		if (found_bbt_pattern(oobbuf, td) || found_bbt_pattern(oobbuf, md)) {
			printk(KERN_INFO "nandCET: %s blk %x is BBT\n", __FUNCTION__, cet->startblk + i*cet->sign);
			cet->flags = NAND_CET_DISABLED;
			return -1;
		}
		/* Check for CET pattern */
		ret = found_cet_pattern(this, oobbuf);
		if (unlikely(gdebug)) {
			print_oobbuf((const char *) oobbuf, mtd->oobsize);
		}
		if (ret < 0 || ret >= cet->numblks) {
			/* No CET pattern found due to
			   1. first time being booted => normal so create
			   2. Did not find CET pattern when we're supposed to
			      error => recreate, in either case we call create_cet_blks();
			   3. Found an incorrect > cet->numblks count => error => recreate
			 */
			printk(KERN_INFO "nandCET: Did not find CET, recreating\n");
			if (create_cet_blks(mtd, cet) < 0) {
				cet->flags = NAND_CET_DISABLED;
				return ret;
			}
			cet->flags = NAND_CET_LAZY;
			return 0;
		}
		/* Found CET pattern */
		if (unlikely(gdebug)) {
			printk(KERN_INFO "nandCET: Found CET block#%d\n", count);
		}
		/* If this is the first block do some extra stuff ... */