drivers/mtd/nand/raw/nand_micron.c - third_party/kernel - Git at Google

 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  * Copyright (C) 2017 Free Electrons
  * Copyright (C) 2017 NextThing Co
  *
  * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
  */

 #include <linux/slab.h>

 #include "internals.h"

 /*
  * Special Micron status bit 3 indicates that the block has been
  * corrected by on-die ECC and should be rewritten.
  */
 #define NAND_ECC_STATUS_WRITE_RECOMMENDED	BIT(3)

 /*
  * On chips with 8-bit ECC and additional bit can be used to distinguish
  * cases where a errors were corrected without needing a rewrite
  *
  * Bit 4 Bit 3 Bit 0 Description
  * ----- ----- ----- -----------
  * 0     0     0     No Errors
  * 0     0     1     Multiple uncorrected errors
  * 0     1     0     4 - 6 errors corrected, recommend rewrite
  * 0     1     1     Reserved
  * 1     0     0     1 - 3 errors corrected
  * 1     0     1     Reserved
  * 1     1     0     7 - 8 errors corrected, recommend rewrite
  */
 #define NAND_ECC_STATUS_MASK		(BIT(4) | BIT(3) | BIT(0))
 #define NAND_ECC_STATUS_UNCORRECTABLE	BIT(0)
 #define NAND_ECC_STATUS_4_6_CORRECTED	BIT(3)
 #define NAND_ECC_STATUS_1_3_CORRECTED	BIT(4)
 #define NAND_ECC_STATUS_7_8_CORRECTED	(BIT(4) | BIT(3))

 struct nand_onfi_vendor_micron {
 	u8 two_plane_read;
 	u8 read_cache;
 	u8 read_unique_id;
 	u8 dq_imped;
 	u8 dq_imped_num_settings;
 	u8 dq_imped_feat_addr;
 	u8 rb_pulldown_strength;
 	u8 rb_pulldown_strength_feat_addr;
 	u8 rb_pulldown_strength_num_settings;
 	u8 otp_mode;
 	u8 otp_page_start;
 	u8 otp_data_prot_addr;
 	u8 otp_num_pages;
 	u8 otp_feat_addr;
 	u8 read_retry_options;
 	u8 reserved[72];
 	u8 param_revision;
 } __packed;

 struct micron_on_die_ecc {
 	bool forced;
 	bool enabled;
 	void *rawbuf;
 };

 struct micron_nand {
 	struct micron_on_die_ecc ecc;
 };

 static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode)
 {
 	u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode};

 	return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature);
 }

 /*
  * Configure chip properties from Micron vendor-specific ONFI table
  */
 static int micron_nand_onfi_init(struct nand_chip *chip)
 {
 	struct nand_parameters *p = &chip->parameters;

 	if (p->onfi) {
 		struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor;

 		chip->read_retries = micron->read_retry_options;
 		chip->setup_read_retry = micron_nand_setup_read_retry;
 	}

 	if (p->supports_set_get_features) {
 		set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list);
 		set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list);
 		set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list);
 		set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list);
 	}

 	return 0;
 }

 static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd,
 					      int section,
 					      struct mtd_oob_region *oobregion)
 {
 	if (section >= 4)
 		return -ERANGE;

 	oobregion->offset = (section * 16) + 8;
 	oobregion->length = 8;

 	return 0;
 }

 static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd,
 					       int section,
 					       struct mtd_oob_region *oobregion)
 {
 	if (section >= 4)
 		return -ERANGE;

 	oobregion->offset = (section * 16) + 2;
 	oobregion->length = 6;

 	return 0;
 }

 static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = {
 	.ecc = micron_nand_on_die_4_ooblayout_ecc,
 	.free = micron_nand_on_die_4_ooblayout_free,
 };

 static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd,
 					      int section,
 					      struct mtd_oob_region *oobregion)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);

 	if (section)
 		return -ERANGE;

 	oobregion->offset = mtd->oobsize - chip->ecc.total;
 	oobregion->length = chip->ecc.total;

 	return 0;
 }

 static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd,
 					       int section,
 					       struct mtd_oob_region *oobregion)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);

 	if (section)
 		return -ERANGE;

 	oobregion->offset = 2;
 	oobregion->length = mtd->oobsize - chip->ecc.total - 2;

 	return 0;
 }

 static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = {
 	.ecc = micron_nand_on_die_8_ooblayout_ecc,
 	.free = micron_nand_on_die_8_ooblayout_free,
 };

 static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable)
 {
 	struct micron_nand *micron = nand_get_manufacturer_data(chip);
 	u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, };
 	int ret;

 	if (micron->ecc.forced)
 		return 0;

 	if (micron->ecc.enabled == enable)
 		return 0;

 	if (enable)
 		feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN;

 	ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature);
 	if (!ret)
 		micron->ecc.enabled = enable;

 	return ret;
 }

 static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status,
 					   void *buf, int page,
 					   int oob_required)
 {
 	struct micron_nand *micron = nand_get_manufacturer_data(chip);
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	unsigned int step, max_bitflips = 0;
 	int ret;

 	if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) {
 		if (status & NAND_STATUS_FAIL)
 			mtd->ecc_stats.failed++;

 		return 0;
 	}

 	/*
 	 * The internal ECC doesn't tell us the number of bitflips that have
 	 * been corrected, but tells us if it recommends to rewrite the block.
 	 * If it's the case, we need to read the page in raw mode and compare
 	 * its content to the corrected version to extract the actual number of
 	 * bitflips.
 	 * But before we do that, we must make sure we have all OOB bytes read
 	 * in non-raw mode, even if the user did not request those bytes.
 	 */
 	if (!oob_required) {
 		ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
 					false);
 		if (ret)
 			return ret;
 	}

 	micron_nand_on_die_ecc_setup(chip, false);

 	ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf,
 				mtd->writesize + mtd->oobsize);
 	if (ret)
 		return ret;

 	for (step = 0; step < chip->ecc.steps; step++) {
 		unsigned int offs, i, nbitflips = 0;
 		u8 *rawbuf, *corrbuf;

 		offs = step * chip->ecc.size;
 		rawbuf = micron->ecc.rawbuf + offs;
 		corrbuf = buf + offs;

 		for (i = 0; i < chip->ecc.size; i++)
 			nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);

 		offs = (step * 16) + 4;
 		rawbuf = micron->ecc.rawbuf + mtd->writesize + offs;
 		corrbuf = chip->oob_poi + offs;

 		for (i = 0; i < chip->ecc.bytes + 4; i++)
 			nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);

 		if (WARN_ON(nbitflips > chip->ecc.strength))
 			return -EINVAL;

 		max_bitflips = max(nbitflips, max_bitflips);
 		mtd->ecc_stats.corrected += nbitflips;
 	}

 	return max_bitflips;
 }

 static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);

 	/*
 	 * With 8/512 we have more information but still don't know precisely
 	 * how many bit-flips were seen.
 	 */
 	switch (status & NAND_ECC_STATUS_MASK) {
 	case NAND_ECC_STATUS_UNCORRECTABLE:
 		mtd->ecc_stats.failed++;
 		return 0;
 	case NAND_ECC_STATUS_1_3_CORRECTED:
 		mtd->ecc_stats.corrected += 3;
 		return 3;
 	case NAND_ECC_STATUS_4_6_CORRECTED:
 		mtd->ecc_stats.corrected += 6;
 		/* rewrite recommended */
 		return 6;
 	case NAND_ECC_STATUS_7_8_CORRECTED:
 		mtd->ecc_stats.corrected += 8;
 		/* rewrite recommended */
 		return 8;
 	default:
 		return 0;
 	}
 }

 static int
 micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf,
 				 int oob_required, int page)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	u8 status;
 	int ret, max_bitflips = 0;

 	ret = micron_nand_on_die_ecc_setup(chip, true);
 	if (ret)
 		return ret;

 	ret = nand_read_page_op(chip, page, 0, NULL, 0);
 	if (ret)
 		goto out;

 	ret = nand_status_op(chip, &status);
 	if (ret)
 		goto out;

 	ret = nand_exit_status_op(chip);
 	if (ret)
 		goto out;

 	ret = nand_read_data_op(chip, buf, mtd->writesize, false);
 	if (!ret && oob_required)
 		ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
 					false);

 	if (chip->ecc.strength == 4)
 		max_bitflips = micron_nand_on_die_ecc_status_4(chip, status,
 							       buf, page,
 							       oob_required);
 	else
 		max_bitflips = micron_nand_on_die_ecc_status_8(chip, status);

 out:
 	micron_nand_on_die_ecc_setup(chip, false);

 	return ret ? ret : max_bitflips;
 }

 static int
 micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf,
 				  int oob_required, int page)
 {
 	int ret;

 	ret = micron_nand_on_die_ecc_setup(chip, true);
 	if (ret)
 		return ret;

 	ret = nand_write_page_raw(chip, buf, oob_required, page);
 	micron_nand_on_die_ecc_setup(chip, false);

 	return ret;
 }

 enum {
 	/* The NAND flash doesn't support on-die ECC */
 	MICRON_ON_DIE_UNSUPPORTED,

 	/*
 	 * The NAND flash supports on-die ECC and it can be
 	 * enabled/disabled by a set features command.
 	 */
 	MICRON_ON_DIE_SUPPORTED,

 	/*
 	 * The NAND flash supports on-die ECC, and it cannot be
 	 * disabled.
 	 */
 	MICRON_ON_DIE_MANDATORY,
 };

 #define MICRON_ID_INTERNAL_ECC_MASK	GENMASK(1, 0)
 #define MICRON_ID_ECC_ENABLED		BIT(7)

 /*
  * Try to detect if the NAND support on-die ECC. To do this, we enable
  * the feature, and read back if it has been enabled as expected. We
  * also check if it can be disabled, because some Micron NANDs do not
  * allow disabling the on-die ECC and we don't support such NANDs for
  * now.
  *
  * This function also has the side effect of disabling on-die ECC if
  * it had been left enabled by the firmware/bootloader.
  */
 static int micron_supports_on_die_ecc(struct nand_chip *chip)
 {
 	u8 id[5];
 	int ret;

 	if (!chip->parameters.onfi)
 		return MICRON_ON_DIE_UNSUPPORTED;

 	if (nanddev_bits_per_cell(&chip->base) != 1)
 		return MICRON_ON_DIE_UNSUPPORTED;

 	/*
 	 * We only support on-die ECC of 4/512 or 8/512
 	 */
 	if  (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8)
 		return MICRON_ON_DIE_UNSUPPORTED;

 	/* 0x2 means on-die ECC is available. */
 	if (chip->id.len != 5 ||
 	    (chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2)
 		return MICRON_ON_DIE_UNSUPPORTED;

 	/*
 	 * It seems that there are devices which do not support ECC officially.
 	 * At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports
 	 * enabling the ECC feature but don't reflect that to the READ_ID table.
 	 * So we have to guarantee that we disable the ECC feature directly
 	 * after we did the READ_ID table command. Later we can evaluate the
 	 * ECC_ENABLE support.
 	 */
 	ret = micron_nand_on_die_ecc_setup(chip, true);
 	if (ret)
 		return MICRON_ON_DIE_UNSUPPORTED;

 	ret = nand_readid_op(chip, 0, id, sizeof(id));
 	if (ret)
 		return MICRON_ON_DIE_UNSUPPORTED;

 	ret = micron_nand_on_die_ecc_setup(chip, false);
 	if (ret)
 		return MICRON_ON_DIE_UNSUPPORTED;

 	if (!(id[4] & MICRON_ID_ECC_ENABLED))
 		return MICRON_ON_DIE_UNSUPPORTED;

 	ret = nand_readid_op(chip, 0, id, sizeof(id));
 	if (ret)
 		return MICRON_ON_DIE_UNSUPPORTED;

 	if (id[4] & MICRON_ID_ECC_ENABLED)
 		return MICRON_ON_DIE_MANDATORY;

 	/*
 	 * We only support on-die ECC of 4/512 or 8/512
 	 */
 	if  (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8)
 		return MICRON_ON_DIE_UNSUPPORTED;

 	return MICRON_ON_DIE_SUPPORTED;
 }

 static int micron_nand_init(struct nand_chip *chip)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	struct micron_nand *micron;
 	int ondie;
 	int ret;

 	micron = kzalloc(sizeof(*micron), GFP_KERNEL);
 	if (!micron)
 		return -ENOMEM;

 	nand_set_manufacturer_data(chip, micron);

 	ret = micron_nand_onfi_init(chip);
 	if (ret)
 		goto err_free_manuf_data;

 	chip->options |= NAND_BBM_FIRSTPAGE;

 	if (mtd->writesize == 2048)
 		chip->options |= NAND_BBM_SECONDPAGE;

 	ondie = micron_supports_on_die_ecc(chip);

 	if (ondie == MICRON_ON_DIE_MANDATORY &&
 	    chip->ecc.mode != NAND_ECC_ON_DIE) {
 		pr_err("On-die ECC forcefully enabled, not supported\n");
 		ret = -EINVAL;
 		goto err_free_manuf_data;
 	}

 	if (chip->ecc.mode == NAND_ECC_ON_DIE) {
 		if (ondie == MICRON_ON_DIE_UNSUPPORTED) {
 			pr_err("On-die ECC selected but not supported\n");
 			ret = -EINVAL;
 			goto err_free_manuf_data;
 		}

 		if (ondie == MICRON_ON_DIE_MANDATORY) {
 			micron->ecc.forced = true;
 			micron->ecc.enabled = true;
 		}

 		/*
 		 * In case of 4bit on-die ECC, we need a buffer to store a
 		 * page dumped in raw mode so that we can compare its content
 		 * to the same page after ECC correction happened and extract
 		 * the real number of bitflips from this comparison.
 		 * That's not needed for 8-bit ECC, because the status expose
 		 * a better approximation of the number of bitflips in a page.
 		 */
 		if (chip->base.eccreq.strength == 4) {
 			micron->ecc.rawbuf = kmalloc(mtd->writesize +
 						     mtd->oobsize,
 						     GFP_KERNEL);
 			if (!micron->ecc.rawbuf) {
 				ret = -ENOMEM;
 				goto err_free_manuf_data;
 			}
 		}

 		if (chip->base.eccreq.strength == 4)
 			mtd_set_ooblayout(mtd,
 					  &micron_nand_on_die_4_ooblayout_ops);
 		else
 			mtd_set_ooblayout(mtd,
 					  &micron_nand_on_die_8_ooblayout_ops);

 		chip->ecc.bytes = chip->base.eccreq.strength * 2;
 		chip->ecc.size = 512;
 		chip->ecc.strength = chip->base.eccreq.strength;
 		chip->ecc.algo = NAND_ECC_BCH;
 		chip->ecc.read_page = micron_nand_read_page_on_die_ecc;
 		chip->ecc.write_page = micron_nand_write_page_on_die_ecc;

 		if (ondie == MICRON_ON_DIE_MANDATORY) {
 			chip->ecc.read_page_raw = nand_read_page_raw_notsupp;
 			chip->ecc.write_page_raw = nand_write_page_raw_notsupp;
 		} else {
 			chip->ecc.read_page_raw = nand_read_page_raw;
 			chip->ecc.write_page_raw = nand_write_page_raw;
 		}
 	}

 	return 0;

 err_free_manuf_data:
 	kfree(micron->ecc.rawbuf);
 	kfree(micron);

 	return ret;
 }

 static void micron_nand_cleanup(struct nand_chip *chip)
 {
 	struct micron_nand *micron = nand_get_manufacturer_data(chip);

 	kfree(micron->ecc.rawbuf);
 	kfree(micron);
 }

 static void micron_fixup_onfi_param_page(struct nand_chip *chip,
 					 struct nand_onfi_params *p)
 {
 	/*
 	 * MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the
 	 * revision number field of the ONFI parameter page. Assume ONFI
 	 * version 1.0 if the revision number is 00 00.
 	 */
 	if (le16_to_cpu(p->revision) == 0)
 		p->revision = cpu_to_le16(ONFI_VERSION_1_0);
 }

 const struct nand_manufacturer_ops micron_nand_manuf_ops = {
 	.init = micron_nand_init,
 	.cleanup = micron_nand_cleanup,
 	.fixup_onfi_param_page = micron_fixup_onfi_param_page,
 };
	// SPDX-License-Identifier: GPL-2.0-or-later
	/*
	* Copyright (C) 2017 Free Electrons
	* Copyright (C) 2017 NextThing Co
	*
	* Author: Boris Brezillon <boris.brezillon@free-electrons.com>
	*/

	#include <linux/slab.h>

	#include "internals.h"

	/*
	* Special Micron status bit 3 indicates that the block has been
	* corrected by on-die ECC and should be rewritten.
	*/
	#define NAND_ECC_STATUS_WRITE_RECOMMENDED BIT(3)

	/*
	* On chips with 8-bit ECC and additional bit can be used to distinguish
	* cases where a errors were corrected without needing a rewrite
	*
	* Bit 4 Bit 3 Bit 0 Description
	* ----- ----- ----- -----------
	* 0 0 0 No Errors
	* 0 0 1 Multiple uncorrected errors
	* 0 1 0 4 - 6 errors corrected, recommend rewrite
	* 0 1 1 Reserved
	* 1 0 0 1 - 3 errors corrected
	* 1 0 1 Reserved
	* 1 1 0 7 - 8 errors corrected, recommend rewrite
	*/
	#define NAND_ECC_STATUS_MASK (BIT(4) \| BIT(3) \| BIT(0))
	#define NAND_ECC_STATUS_UNCORRECTABLE BIT(0)
	#define NAND_ECC_STATUS_4_6_CORRECTED BIT(3)
	#define NAND_ECC_STATUS_1_3_CORRECTED BIT(4)
	#define NAND_ECC_STATUS_7_8_CORRECTED (BIT(4) \| BIT(3))

	struct nand_onfi_vendor_micron {
	u8 two_plane_read;
	u8 read_cache;
	u8 read_unique_id;
	u8 dq_imped;
	u8 dq_imped_num_settings;
	u8 dq_imped_feat_addr;
	u8 rb_pulldown_strength;
	u8 rb_pulldown_strength_feat_addr;
	u8 rb_pulldown_strength_num_settings;
	u8 otp_mode;
	u8 otp_page_start;
	u8 otp_data_prot_addr;
	u8 otp_num_pages;
	u8 otp_feat_addr;
	u8 read_retry_options;
	u8 reserved[72];
	u8 param_revision;
	} __packed;

	struct micron_on_die_ecc {
	bool forced;
	bool enabled;
	void *rawbuf;
	};

	struct micron_nand {
	struct micron_on_die_ecc ecc;
	};

	static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode)
	{
	u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode};

	return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature);
	}

	/*
	* Configure chip properties from Micron vendor-specific ONFI table
	*/
	static int micron_nand_onfi_init(struct nand_chip *chip)
	{
	struct nand_parameters *p = &chip->parameters;

	if (p->onfi) {
	struct nand_onfi_vendor_micron micron = (void )p->onfi->vendor;

	chip->read_retries = micron->read_retry_options;
	chip->setup_read_retry = micron_nand_setup_read_retry;
	}

	if (p->supports_set_get_features) {
	set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list);
	set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list);
	set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list);
	set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list);
	}

	return 0;
	}

	static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd,
	int section,
	struct mtd_oob_region *oobregion)
	{
	if (section >= 4)
	return -ERANGE;

	oobregion->offset = (section * 16) + 8;
	oobregion->length = 8;

	return 0;
	}

	static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd,
	int section,
	struct mtd_oob_region *oobregion)
	{
	if (section >= 4)
	return -ERANGE;

	oobregion->offset = (section * 16) + 2;
	oobregion->length = 6;

	return 0;
	}

	static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = {
	.ecc = micron_nand_on_die_4_ooblayout_ecc,
	.free = micron_nand_on_die_4_ooblayout_free,
	};

	static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd,
	int section,
	struct mtd_oob_region *oobregion)
	{
	struct nand_chip *chip = mtd_to_nand(mtd);

	if (section)
	return -ERANGE;

	oobregion->offset = mtd->oobsize - chip->ecc.total;
	oobregion->length = chip->ecc.total;

	return 0;
	}

	static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd,
	int section,
	struct mtd_oob_region *oobregion)
	{
	struct nand_chip *chip = mtd_to_nand(mtd);

	if (section)
	return -ERANGE;

	oobregion->offset = 2;
	oobregion->length = mtd->oobsize - chip->ecc.total - 2;

	return 0;
	}

	static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = {
	.ecc = micron_nand_on_die_8_ooblayout_ecc,
	.free = micron_nand_on_die_8_ooblayout_free,
	};

	static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable)
	{
	struct micron_nand *micron = nand_get_manufacturer_data(chip);
	u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, };
	int ret;

	if (micron->ecc.forced)
	return 0;

	if (micron->ecc.enabled == enable)
	return 0;

	if (enable)
	feature[0] \|= ONFI_FEATURE_ON_DIE_ECC_EN;

	ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature);
	if (!ret)
	micron->ecc.enabled = enable;

	return ret;
	}

	static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status,
	void *buf, int page,
	int oob_required)
	{
	struct micron_nand *micron = nand_get_manufacturer_data(chip);
	struct mtd_info *mtd = nand_to_mtd(chip);
	unsigned int step, max_bitflips = 0;
	int ret;

	if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) {
	if (status & NAND_STATUS_FAIL)
	mtd->ecc_stats.failed++;

	return 0;
	}

	/*
	* The internal ECC doesn't tell us the number of bitflips that have
	* been corrected, but tells us if it recommends to rewrite the block.
	* If it's the case, we need to read the page in raw mode and compare
	* its content to the corrected version to extract the actual number of
	* bitflips.
	* But before we do that, we must make sure we have all OOB bytes read
	* in non-raw mode, even if the user did not request those bytes.
	*/
	if (!oob_required) {
	ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
	false);
	if (ret)
	return ret;
	}

	micron_nand_on_die_ecc_setup(chip, false);

	ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf,
	mtd->writesize + mtd->oobsize);
	if (ret)
	return ret;

	for (step = 0; step < chip->ecc.steps; step++) {
	unsigned int offs, i, nbitflips = 0;
	u8 rawbuf, corrbuf;

	offs = step * chip->ecc.size;
	rawbuf = micron->ecc.rawbuf + offs;
	corrbuf = buf + offs;

	for (i = 0; i < chip->ecc.size; i++)
	nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);

	offs = (step * 16) + 4;
	rawbuf = micron->ecc.rawbuf + mtd->writesize + offs;
	corrbuf = chip->oob_poi + offs;

	for (i = 0; i < chip->ecc.bytes + 4; i++)
	nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);

	if (WARN_ON(nbitflips > chip->ecc.strength))
	return -EINVAL;

	max_bitflips = max(nbitflips, max_bitflips);
	mtd->ecc_stats.corrected += nbitflips;
	}

	return max_bitflips;
	}

	static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status)
	{
	struct mtd_info *mtd = nand_to_mtd(chip);

	/*
	* With 8/512 we have more information but still don't know precisely
	* how many bit-flips were seen.
	*/
	switch (status & NAND_ECC_STATUS_MASK) {
	case NAND_ECC_STATUS_UNCORRECTABLE:
	mtd->ecc_stats.failed++;
	return 0;
	case NAND_ECC_STATUS_1_3_CORRECTED:
	mtd->ecc_stats.corrected += 3;
	return 3;
	case NAND_ECC_STATUS_4_6_CORRECTED:
	mtd->ecc_stats.corrected += 6;
	/* rewrite recommended */
	return 6;
	case NAND_ECC_STATUS_7_8_CORRECTED:
	mtd->ecc_stats.corrected += 8;
	/* rewrite recommended */
	return 8;
	default:
	return 0;
	}
	}

	static int
	micron_nand_read_page_on_die_ecc(struct nand_chip chip, uint8_t buf,
	int oob_required, int page)
	{
	struct mtd_info *mtd = nand_to_mtd(chip);
	u8 status;
	int ret, max_bitflips = 0;

	ret = micron_nand_on_die_ecc_setup(chip, true);
	if (ret)
	return ret;

	ret = nand_read_page_op(chip, page, 0, NULL, 0);
	if (ret)
	goto out;

	ret = nand_status_op(chip, &status);
	if (ret)
	goto out;

	ret = nand_exit_status_op(chip);
	if (ret)
	goto out;

	ret = nand_read_data_op(chip, buf, mtd->writesize, false);
	if (!ret && oob_required)
	ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
	false);

	if (chip->ecc.strength == 4)
	max_bitflips = micron_nand_on_die_ecc_status_4(chip, status,
	buf, page,
	oob_required);
	else
	max_bitflips = micron_nand_on_die_ecc_status_8(chip, status);

	out:
	micron_nand_on_die_ecc_setup(chip, false);

	return ret ? ret : max_bitflips;
	}

	static int
	micron_nand_write_page_on_die_ecc(struct nand_chip chip, const uint8_t buf,
	int oob_required, int page)
	{
	int ret;

	ret = micron_nand_on_die_ecc_setup(chip, true);
	if (ret)
	return ret;

	ret = nand_write_page_raw(chip, buf, oob_required, page);
	micron_nand_on_die_ecc_setup(chip, false);

	return ret;
	}

	enum {
	/* The NAND flash doesn't support on-die ECC */
	MICRON_ON_DIE_UNSUPPORTED,

	/*
	* The NAND flash supports on-die ECC and it can be
	* enabled/disabled by a set features command.
	*/
	MICRON_ON_DIE_SUPPORTED,

	/*
	* The NAND flash supports on-die ECC, and it cannot be
	* disabled.
	*/
	MICRON_ON_DIE_MANDATORY,
	};

	#define MICRON_ID_INTERNAL_ECC_MASK GENMASK(1, 0)
	#define MICRON_ID_ECC_ENABLED BIT(7)

	/*
	* Try to detect if the NAND support on-die ECC. To do this, we enable
	* the feature, and read back if it has been enabled as expected. We
	* also check if it can be disabled, because some Micron NANDs do not
	* allow disabling the on-die ECC and we don't support such NANDs for
	* now.
	*
	* This function also has the side effect of disabling on-die ECC if
	* it had been left enabled by the firmware/bootloader.
	*/
	static int micron_supports_on_die_ecc(struct nand_chip *chip)
	{
	u8 id[5];
	int ret;

	if (!chip->parameters.onfi)
	return MICRON_ON_DIE_UNSUPPORTED;

	if (nanddev_bits_per_cell(&chip->base) != 1)
	return MICRON_ON_DIE_UNSUPPORTED;

	/*
	* We only support on-die ECC of 4/512 or 8/512
	*/
	if (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8)
	return MICRON_ON_DIE_UNSUPPORTED;

	/* 0x2 means on-die ECC is available. */
	if (chip->id.len != 5 \|\|
	(chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2)
	return MICRON_ON_DIE_UNSUPPORTED;

	/*
	* It seems that there are devices which do not support ECC officially.
	* At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports
	* enabling the ECC feature but don't reflect that to the READ_ID table.
	* So we have to guarantee that we disable the ECC feature directly
	* after we did the READ_ID table command. Later we can evaluate the
	* ECC_ENABLE support.
	*/
	ret = micron_nand_on_die_ecc_setup(chip, true);
	if (ret)
	return MICRON_ON_DIE_UNSUPPORTED;

	ret = nand_readid_op(chip, 0, id, sizeof(id));
	if (ret)
	return MICRON_ON_DIE_UNSUPPORTED;

	ret = micron_nand_on_die_ecc_setup(chip, false);
	if (ret)
	return MICRON_ON_DIE_UNSUPPORTED;

	if (!(id[4] & MICRON_ID_ECC_ENABLED))
	return MICRON_ON_DIE_UNSUPPORTED;

	ret = nand_readid_op(chip, 0, id, sizeof(id));
	if (ret)
	return MICRON_ON_DIE_UNSUPPORTED;

	if (id[4] & MICRON_ID_ECC_ENABLED)
	return MICRON_ON_DIE_MANDATORY;

	/*
	* We only support on-die ECC of 4/512 or 8/512
	*/
	if (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8)
	return MICRON_ON_DIE_UNSUPPORTED;

	return MICRON_ON_DIE_SUPPORTED;
	}

	static int micron_nand_init(struct nand_chip *chip)
	{
	struct mtd_info *mtd = nand_to_mtd(chip);
	struct micron_nand *micron;
	int ondie;
	int ret;

	micron = kzalloc(sizeof(*micron), GFP_KERNEL);
	if (!micron)
	return -ENOMEM;

	nand_set_manufacturer_data(chip, micron);

	ret = micron_nand_onfi_init(chip);
	if (ret)
	goto err_free_manuf_data;

	chip->options \|= NAND_BBM_FIRSTPAGE;

	if (mtd->writesize == 2048)
	chip->options \|= NAND_BBM_SECONDPAGE;

	ondie = micron_supports_on_die_ecc(chip);

	if (ondie == MICRON_ON_DIE_MANDATORY &&
	chip->ecc.mode != NAND_ECC_ON_DIE) {
	pr_err("On-die ECC forcefully enabled, not supported\n");
	ret = -EINVAL;
	goto err_free_manuf_data;
	}

	if (chip->ecc.mode == NAND_ECC_ON_DIE) {
	if (ondie == MICRON_ON_DIE_UNSUPPORTED) {
	pr_err("On-die ECC selected but not supported\n");
	ret = -EINVAL;
	goto err_free_manuf_data;
	}

	if (ondie == MICRON_ON_DIE_MANDATORY) {
	micron->ecc.forced = true;
	micron->ecc.enabled = true;
	}

	/*
	* In case of 4bit on-die ECC, we need a buffer to store a
	* page dumped in raw mode so that we can compare its content
	* to the same page after ECC correction happened and extract
	* the real number of bitflips from this comparison.
	* That's not needed for 8-bit ECC, because the status expose
	* a better approximation of the number of bitflips in a page.
	*/
	if (chip->base.eccreq.strength == 4) {
	micron->ecc.rawbuf = kmalloc(mtd->writesize +
	mtd->oobsize,
	GFP_KERNEL);
	if (!micron->ecc.rawbuf) {
	ret = -ENOMEM;
	goto err_free_manuf_data;
	}
	}

	if (chip->base.eccreq.strength == 4)
	mtd_set_ooblayout(mtd,
	&micron_nand_on_die_4_ooblayout_ops);
	else
	mtd_set_ooblayout(mtd,
	&micron_nand_on_die_8_ooblayout_ops);

	chip->ecc.bytes = chip->base.eccreq.strength * 2;
	chip->ecc.size = 512;
	chip->ecc.strength = chip->base.eccreq.strength;
	chip->ecc.algo = NAND_ECC_BCH;
	chip->ecc.read_page = micron_nand_read_page_on_die_ecc;
	chip->ecc.write_page = micron_nand_write_page_on_die_ecc;

	if (ondie == MICRON_ON_DIE_MANDATORY) {
	chip->ecc.read_page_raw = nand_read_page_raw_notsupp;
	chip->ecc.write_page_raw = nand_write_page_raw_notsupp;
	} else {
	chip->ecc.read_page_raw = nand_read_page_raw;
	chip->ecc.write_page_raw = nand_write_page_raw;
	}
	}

	return 0;

	err_free_manuf_data:
	kfree(micron->ecc.rawbuf);
	kfree(micron);

	return ret;
	}

	static void micron_nand_cleanup(struct nand_chip *chip)
	{
	struct micron_nand *micron = nand_get_manufacturer_data(chip);

	kfree(micron->ecc.rawbuf);
	kfree(micron);
	}

	static void micron_fixup_onfi_param_page(struct nand_chip *chip,
	struct nand_onfi_params *p)
	{
	/*
	* MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the
	* revision number field of the ONFI parameter page. Assume ONFI
	* version 1.0 if the revision number is 00 00.
	*/
	if (le16_to_cpu(p->revision) == 0)
	p->revision = cpu_to_le16(ONFI_VERSION_1_0);
	}

	const struct nand_manufacturer_ops micron_nand_manuf_ops = {
	.init = micron_nand_init,
	.cleanup = micron_nand_cleanup,
	.fixup_onfi_param_page = micron_fixup_onfi_param_page,
	};