/**
 * \file
 *
 * \brief SD/MMC MCI host driver
 *
 * SD/MMC MCI host driver providing interface between host layer and hardware.
 *
 * - Compiler:           IAR EWAVR32 and GNU GCC for AVR32
 * - Supported devices:  All devices
 * - AppNote:
 *
 * \author               Atmel Corporation: http://www.atmel.com \n
 *                       Support and FAQ: http://support.atmel.no/
 *
 * \page License
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3. The name of Atmel may not be used to endorse or promote products derived
 * from this software without specific prior written permission.
 *
 * 4. This software may only be redistributed and used in connection with an
 * Atmel AVR product.
 *
 * THIS SOFTWARE IS PROVIDED BY ATMEL "AS IS" AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE
 * EXPRESSLY AND SPECIFICALLY DISCLAIMED. IN NO EVENT SHALL ATMEL BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 */
#include <stdbool.h>
#include <string.h>

#include <debug.h>
#include <delayed_work.h>
#include <bitops.h>
#include <io.h>
#include <interrupt.h>
#include <irq_handler.h>
#include <softirq.h>
#include <status-codes.h>
#include <util.h>
#include <slist.h>
#include <timeout.h>
#include <workqueue.h>
#include <buffer.h>
#include <malloc.h>

#include <dmac/dma_controller.h>
#include <chip/clk.h>
#include <chip/portmux.h>
#include <chip/irq-map.h>
#include <sdmmc/sdmmc.h>
#include <sdmmc/mcihost.h>
#include "protocol.h"
#include "mci_regs.h"
#include <board/sdmmc.h>
#include <app/softirq.h>
#include <app/timer.h>
#include <app/workqueue.h>

#if defined(CONFIG_SDMMC_MCIHOST_SLOT_A) && defined(CONFIG_SDMMC_MCIHOST_SLOT_B)
#define SDMMC_SLOT_COUNT 2
#elif defined(CONFIG_SDMMC_MCIHOST_SLOT_A) || defined(CONFIG_SDMMC_MCIHOST_SLOT_B)
#define SDMMC_SLOT_COUNT 1
#else
#error "No slot config set!"
#endif

/* If a request isn't done after one second, something is wrong */
#define MCIHOST_REQ_TIMEOUT_MS		10000

/* Check for request timeouts every second */
#define MCIHOST_WATCHDOG_PERIOD_MS	10000

enum sdmmc_mcihost_event {
	EVENT_CMD_COMPLETE,
	EVENT_DATA_COMPLETE,
	EVENT_REQ_COMPLETE,
	EVENT_DATA_ERROR,
	EVENT_DMA_COMPLETE,
};

enum sdmmc_mcihost_state {
	STATE_IDLE		= 0,	//!< Initial state
	STATE_SENDING_CMD,		//!< Sending initial command
	STATE_DATA_XFER,		//!< Data transfer to/from card
	STATE_SENDING_STOP,		//!< Sending STOP command
	STATE_FINISH_REQ,		//!< Waiting for XFRDONE
	STATE_CARD_STATUS,		//!< Check card status
};

struct sdmmc_mci_slot {
	uint32_t		dtor_write;
	uint32_t		dtor_read;
	uint32_t		sdc_reg;
	uint32_t		cfg_reg;
	struct sdmmc_slot	slot;
	uint32_t		last_f_max;
};

struct sdmmc_mcihost {
	struct sdmmc_host	host;
	struct dmac_channel	*dmac_chan;
	struct dmac_request	dmac_req;
	struct timeout		req_timeout;
	struct slist		req_queue;
	struct sdmmc_request	*current;
	unsigned long		pending_events;
	enum sdmmc_mcihost_state state;
	struct sdmmc_mci_slot	slots[SDMMC_SLOT_COUNT];
	bool			dtor_valid;
	uint32_t		cmd_status;
	uint32_t		data_status;
	struct workqueue_item	watchdog_work;
	struct delayed_work	watchdog_dw;
};

/* Collection of bits that are cleared on read */
#define MCI_CLEAR_ON_READ_BITS			\
	(MCI_DCRCE | MCI_DTOE | MCI_CSTOE | MCI_BLKOVRE)

enum mci_send_cmd_flags {
	MCI_SEND_CMD_STOP,	/**< CMD is a STOP command */
	MCI_SEND_CMD_WRITE,	/**< CMD is a write command */
};

enum sdmmc_mcihost_hwslot {
	SDMMC_MCIHOST_SLOT_A,
	SDMMC_MCIHOST_SLOT_B,
};

static inline struct sdmmc_mcihost *sdmmc_mcihost_of(struct sdmmc_host *host)
{
	return container_of(host, struct sdmmc_mcihost, host);
}

static inline struct sdmmc_mci_slot *sdmmc_mcislot_of(struct sdmmc_slot *slot)
{
	return container_of(slot, struct sdmmc_mci_slot, slot);
}

static void mcihost_reset_controller(struct sdmmc_mcihost *mcihost)
{
	uint32_t	mr;
	unsigned long	iflags;

	iflags = cpu_irq_save();
	mr = mci_readl(MR);
	mci_writel(CR, MCI_CR_SWRST);
	mci_writel(MR, mr);
	mci_writel(CR, MCI_CR_MCIEN);

	/* Make sure all writes hit the device before we continue */
	mci_readl(MR);
	cpu_irq_restore(iflags);
}

static inline enum sdmmc_mcihost_hwslot sdmmc_mcihost_get_hwslot(int slot_id)
{
#if defined(CONFIG_SDMMC_MCIHOST_SLOT_A) && defined(CONFIG_SDMMC_MCIHOST_SLOT_B)
	switch (slot_id) {
	case 0:
		return SDMMC_MCIHOST_SLOT_A;
	case 1:
		return SDMMC_MCIHOST_SLOT_B;
	default:
		unhandled_case(slot_id);
		return -1;
	}
#elif defined(CONFIG_SDMMC_MCIHOST_SLOT_A)
	return SDMMC_MCIHOST_SLOT_A;
#elif defined(CONFIG_SDMMC_MCIHOST_SLOT_B)
	return SDMMC_MCIHOST_SLOT_B;
#endif
}

static void sdmmc_mcihost_send_cmd(struct sdmmc_mcihost *mcihost,
		struct sdmmc_command *cmd, uint32_t blocks,
		uint32_t block_size, unsigned long flags)
{
	uint32_t		cmdr;

	mcihost->cmd_status = 0;

	cmdr = MCI_CMDR_CMDNB(cmd->opcode);
	if (cmd->flags & SDMMC_RSP_PRESENT) {
		if (cmd->flags & SDMMC_RSP_136)
			cmdr |= MCI_CMDR_RSPTYP_136BIT;
		else if (cmd->flags & SDMMC_RSP_BUSY)
			cmdr |= MCI_CMDR_RSPTYP_R1B;
		else
			cmdr |= MCI_CMDR_RSPTYP_48BIT;
	}
	if (cmd->flags & SDMMC_CMD_OPD)
		cmdr |= MCI_CMDR_OPDCMD;

	cmdr |= MCI_CMDR_MAXLAT_64CYC;

	if (blocks) {
		if (test_bit(MCI_SEND_CMD_STOP, &flags)) {
			cmdr |= MCI_CMDR_STOP_XFER;
		} else {
			cmdr |= MCI_CMDR_START_XFER;
			mci_writel(BLKR, MCI_BLKLEN(block_size)
					| MCI_BCNT(blocks));
		}
		if (blocks > 1)
			cmdr |= MCI_CMDR_MULTI_BLOCK;
		else
			cmdr |= MCI_CMDR_BLOCK;
		if (test_bit(MCI_SEND_CMD_WRITE, &flags))
			cmdr |= MCI_CMDR_TRDIR_WRITE;
		else
			cmdr |= MCI_CMDR_TRDIR_READ;
	}

	mci_writel(ARGR, cmd->arg);
	mci_writel(CMDR, cmdr);
}

static void sdmmc_mcihost_send_initseq(void)
{
	mci_writel(CMDR, MCI_CMDR_SPCMD_INIT);
}

static inline void sdmmc_mcihost_send_stop(struct sdmmc_mcihost *mcihost,
		bool write)
{
	uint32_t	cmdr;

	mcihost->cmd_status = 0;

	cmdr = MCI_CMDR_CMDNB(SDMMC_STOP_TRANSMISSION) |
		MCI_CMDR_RSPTYP_R1B |
		MCI_CMDR_MAXLAT_64CYC |
		MCI_CMDR_STOP_XFER |
		MCI_CMDR_MULTI_BLOCK;

	if (write)
		cmdr |= MCI_CMDR_TRDIR_WRITE;
	else
		cmdr |= MCI_CMDR_TRDIR_READ;

	mci_writel(ARGR, 0);
	mci_writel(CMDR, cmdr);
	mci_writel(IER, MCI_CMDRDY);
}

static uint32_t sdmmc_mcihost_calc_data_timeout(struct sdmmc_mcihost *mcihost,
		uint32_t timeout_ns, uint32_t timeout_clks)
{
	static const uint32_t dtomul_to_shift[] = {
		0, 4, 7, 8, 10, 12, 16, 20
	};
	uint32_t	bus_hz;
	uint32_t	clocks;
	uint32_t	dtocyc;
	uint32_t	dtomul;

	bus_hz = get_mci_pclk_rate(0);
	clocks = div_ceil(timeout_ns * (bus_hz / 1000000), 1000);
	clocks += timeout_clks;

	for (dtomul = 0; dtomul < 8; dtomul++) {
		unsigned int shift = dtomul_to_shift[dtomul];
		dtocyc = (clocks + (1 << shift) - 1) >> shift;
		if (dtocyc < 15)
			break;
	}

	if (dtomul >= 8) {
		dtomul = 7;
		dtocyc = 15;
	}

	return MCI_DTOMUL(dtomul) | MCI_DTOCYC(dtocyc);
}

static void sdmmc_mcihost_update_timeouts(struct sdmmc_mcihost *mcihost,
		struct sdmmc_mci_slot *mcislot)
{
	struct sdmmc_card		*card = &mcislot->slot.card;

	sdmmc_card_update_timeouts(card, mcihost->host.f_cur);
	mcislot->dtor_read = sdmmc_mcihost_calc_data_timeout(mcihost,
			card->read_timeout_ns, card->read_timeout_clks);
	mcislot->dtor_write = sdmmc_mcihost_calc_data_timeout(mcihost,
			card->write_timeout_ns, card->write_timeout_clks);

	dbg_verbose("sdmmc_mcihost: data timeout: read %u / write %u\n",
			mcislot->dtor_read, mcislot->dtor_write);
}

static void sdmmc_mcihost_set_clk_rate(struct sdmmc_mcihost *mcihost,
		struct sdmmc_mci_slot *mcislot)
{
	struct sdmmc_slot	*slot;
	uint32_t		clkdiv;
	uint32_t		master_hz;
	uint32_t		target_hz;
	int			i;

	if (mcislot->last_f_max == mcislot->slot.f_max)
		return;

	/* Find the lowest rate among all slots that aren't "don't care" */
	target_hz = mcihost->host.f_max;
	for (i = 0; i < SDMMC_SLOT_COUNT; i++) {
		slot = &mcihost->slots[i].slot;
		if (slot->f_max >= 0)
			target_hz = min(target_hz, slot->f_max);
	}

	/* If we're stopping the clock, no need for a new divider. */
	if (target_hz == 0) {
		mcislot->last_f_max = mcislot->slot.f_max;
		dbg_verbose("sdmmc_mcihost: stopping the clock\n");
		mci_writel(CR, MCI_CR_MCIDIS);
		return;
	}

	master_hz = get_mci_pclk_rate(0);
	clkdiv = div_ceil(master_hz, 2 * target_hz) - 1;
	if (clkdiv > 255)
		clkdiv = 255;

	mcihost->host.f_cur = master_hz / (2 * (clkdiv + 1));
	dbg_verbose("sdmmc_mcihost: new clock rate: %u Hz (requested %u Hz)\n",
			mcihost->host.f_cur, target_hz);
	mci_writel(MR, MCI_MR_CLKDIV(clkdiv) | MCI_MR_RDPROOF | MCI_MR_WRPROOF);

	/* If the clock used to be stopped, start it now */
	if (mcislot->last_f_max == 0)
		mci_writel(CR, MCI_CR_MCIEN);

	mcislot->last_f_max = mcislot->slot.f_max;
}

static void sdmmc_mcihost_update_bus_width(struct sdmmc_mci_slot *mcislot)
{
	uint32_t	sdcr;

	sdcr = mcislot->sdc_reg & ~MCI_SDCBUS_MASK;
	switch (mcislot->slot.bus_width) {
	case 1:
		sdcr |= MCI_SDCBUS_1BIT;
		break;
	case 4:
		sdcr |= MCI_SDCBUS_4BIT;
		break;
	case 8:
		sdcr |= MCI_SDCBUS_8BIT;
		break;
	default:
		unhandled_case(mcislot->slot.bus_width);
		break;
	}
	mcislot->sdc_reg = sdcr;

	dbg_verbose("sdmmc_mcihost: current bus width: %u (SDCR=%08x)\n",
			mcislot->slot.bus_width, sdcr);
}

static void sdmmc_mcihost_set_bus_params(struct sdmmc_host *host,
		struct sdmmc_slot *slot)
{
	struct sdmmc_mcihost	*mcihost = sdmmc_mcihost_of(host);
	struct sdmmc_mci_slot	*mcislot = sdmmc_mcislot_of(slot);
	int			i;

	if (test_bit(SDMMC_SLOT_HIGH_SPEED, &mcislot->slot.flags))
		mcislot->cfg_reg |= MCI_CFG_HSMODE;
	else
		mcislot->cfg_reg &= ~MCI_CFG_HSMODE;

	sdmmc_mcihost_update_bus_width(mcislot);
	sdmmc_mcihost_set_clk_rate(mcihost, mcislot);

	/*
	 * The MMC bus clock rate may have changed, so make sure the data
	 * timeouts for all slots are updated.
	 */
	for (i = 0; i < mcihost->host.slot_count; i++)
		sdmmc_mcihost_update_timeouts(mcihost, &mcihost->slots[i]);
}

/* TODO: external timeout for dma transfer (?)
 * TODO: check MCI status for error during dma
 * TODO: see below:
 *
 * Timing is tricky here...
 *
 *   - We can't start transferring data before the command is sent
 *   - We must initialize the DMA register before sending the command
 *   - We must be prepared to handle command failure
 *       o Currently we handle this by waiting for the command to
 *         complete before starting the data transfer.
 *   - When the card recognizes the stop command, it will stop
 *     transferring data. So we can't send it before the FIFO is
 *     drained.
 *   - The "synchronized command" feature was inherently racy last
 *     time I checked -- it would hang forever if the data transfer
 *     finished before the stop command was sent.
 *   - We can still see data error flags after the actual data
 *     transfer is done. So we must check for those until XFRDONE is
 *     set.
 *   - The stop command can fail, but fortunately the command error
 *     flags are not clear-on-read, so we can check for those when
 *     XFRDONE and CMDRDY are both set.
 *   - Clear-on-read is just plain stupid. This code would have been
 *     _much_ simpler if the error bits stuck until the end of the
 *     transfer.
 */

/**
 * \private
 * \brief SD/MMC MCI host send next request in queue
 *
 * \pre Interrupts must be disabled
 */
static void sdmmc_mcihost_send_next_req(struct sdmmc_mcihost *mcihost)
{
	struct sdmmc_mci_slot	*mcislot;
	struct sdmmc_request	*req;
	struct sdmmc_command	*cmd;

	if (unlikely(mcihost->current))
		return;

	if (unlikely(slist_is_empty(&mcihost->req_queue)))
		return;

	req = slist_pop_head(&mcihost->req_queue, struct sdmmc_request, node);
	mcihost->current = req;

	assert(!test_bit(SDMMC_REQ_WRITE, &req->flags) || req->blocks > 0);

	cmd = &req->cmd;

	/* Must initialize the timeout before leaving STATE_IDLE */
	timeout_init_ms(&mcihost->req_timeout, MCIHOST_REQ_TIMEOUT_MS);
	barrier();
	mcihost->state = STATE_SENDING_CMD;

	mcislot = sdmmc_mcislot_of(req->slot);

	/* Set High Speed timing if required */
	mci_writel(CFG, mcislot->cfg_reg);

	/* Select slot and bus width */
	mci_writel(SDCR, mcislot->sdc_reg);

	/* If there is data we need to enable DMA now and set data timeout */
	if (req->blocks) {
		mci_writel(DMA, MCI_DMAEN);
		if (test_bit(SDMMC_REQ_WRITE, &req->flags))
			mci_writel(DTOR, mcislot->dtor_write);
		else
			mci_writel(DTOR, mcislot->dtor_read);
	}

	/* Finally start the command */
	if (test_bit(SDMMC_REQ_INITSEQ, &req->flags)) {
		dbg_verbose("mcihost: Initialization sequence (74 idle "
				"cycles)...\n");
		sdmmc_mcihost_send_initseq();
	} else {
		unsigned long flags;

		dbg_verbose("mcihost: CMD%u(%08x) flags: %08lx\n", cmd->opcode,
				cmd->arg, cmd->flags);

		flags = 0;
		if (test_bit(SDMMC_REQ_WRITE, &req->flags))
			set_bit(MCI_SEND_CMD_WRITE, &flags);

		sdmmc_mcihost_send_cmd(mcihost, cmd, req->blocks,
				req->block_size, flags);
	}
	mci_writel(IER, MCI_CMDRDY);
}

static void sdmmc_mcihost_submit_req(struct sdmmc_host *host,
		struct sdmmc_request *req)
{
	struct sdmmc_mcihost *mcihost = sdmmc_mcihost_of(host);
	unsigned long flags;

	req->status = -STATUS_IN_PROGRESS;
	req->bytes_xfered = 0;

	flags = cpu_irq_save();
	slist_insert_tail(&mcihost->req_queue, &req->node);
	sdmmc_mcihost_send_next_req(mcihost);
	cpu_irq_restore(flags);
}

static void sdmmc_mcihost_set_power(int slot_id, bool power)
{
#if defined(BOARD_MCISLOT_A_PWREN) || defined(BOARD_MCISLOT_B_PWREN)
	switch (sdmmc_mcihost_get_hwslot(slot_id)) {
#ifdef BOARD_MCISLOT_A_PWREN
	case SDMMC_MCIHOST_SLOT_A:
		board_mcislot_a_set_power(power);
		break;
#endif
#ifdef BOARD_MCISLOT_B_PWREN
	case SDMMC_MCIHOST_SLOT_B:
		board_mcislot_b_set_power(power);
		break;
#endif
	}
#endif
}

static void sdmmc_mcihost_power_up(struct sdmmc_host *host,
		struct sdmmc_slot *slot)
{
	sdmmc_mcihost_set_power(slot->id, true);
}

static void sdmmc_mcihost_power_down(struct sdmmc_host *host,
		struct sdmmc_slot *slot)
{
	sdmmc_mcihost_set_power(slot->id, false);
}

static void sdmmc_mcihost_set_voltage(struct sdmmc_host *host,
		uint32_t ocr_bit)
{
	dbg_verbose("mcihost: TODO: set voltage to %u\n", ocr_bit);
}

static void sdmmc_mcihost_req_close(struct sdmmc_mcihost *mcihost)
{
	struct sdmmc_request	*req = mcihost->current;

	if (req->status == -STATUS_IN_PROGRESS)
		req->status = STATUS_OK;

	dbg_verbose("mcihost: request done, status=%d, bytes_xfered: %zu\n",
			req->status, req->bytes_xfered);
	mcihost->current = NULL;
	sdmmc_mcihost_send_next_req(mcihost);
	req->req_done(req);
}

/**
 * \internal
 * \brief Process the DMA queue
 *
 * \note Must be called with softirq disabled
 */
static void sdmmc_mcihost_load_data(struct sdmmc_mcihost *mcihost)
{
	struct sdmmc_request	*sreq;
	struct dmac_request	*dreq;
	struct slist		buf_list;
	size_t			bytes_total;

	dreq = &mcihost->dmac_req;

	slist_init(&buf_list);
	slist_move_to_tail(&buf_list, &dreq->buf_list);

	sreq = mcihost->current;
	sreq->bytes_xfered += dreq->bytes_xfered;

	bytes_total = sreq->blocks * sreq->block_size;
	assert(sreq->bytes_xfered <= bytes_total);

	dbg_verbose("mcihost load_data: ds %d ms %d bx %zu/%zu\n",
			dreq->status, sreq->status, sreq->bytes_xfered,
			bytes_total);

	/*
	 * Don't enable the FIFOEMPTY interrupt if we've detected an MMC
	 * bus error earlier -- in this case, we won't finish the
	 * transfer anyway.
	 */
	if (sreq->status == -STATUS_IN_PROGRESS) {
		if (dreq->status) {
			dbg_error("mcihost: dmac transfer failure: %d\n",
					dreq->status);
			sreq->status = dreq->status;

			mci_writel(IER, MCI_FIFOEMPTY);
		} else if (sreq->bytes_xfered >= bytes_total) {
			assert(slist_is_empty(&sreq->buf_list));

			mci_writel(IER, MCI_FIFOEMPTY);
		} else if (!slist_is_empty(&sreq->buf_list)) {
			slist_move_to_tail(&dreq->buf_list, &sreq->buf_list);
			dmac_chan_submit_request(mcihost->dmac_chan, dreq);
		}
	}

	if (sreq->buf_list_done)
		sreq->buf_list_done(sreq, &buf_list);
	else
		slist_move_to_tail(&sreq->buf_list, &buf_list);
}

static int sdmmc_mcihost_submit_buf_list(struct sdmmc_host *host,
		struct sdmmc_request *req, struct slist *buf_list)
{
	struct sdmmc_mcihost	*mcihost = sdmmc_mcihost_of(host);
	unsigned long		sflags;
	int			ret = 0;

	dbg_verbose("mcihost: submit_buf_list r%p c%p s%d state %d\n",
			req, mcihost->current, req->status, mcihost->state);

	/*
	 * This function will race with the softirq handler if called
	 * from a hardirq handler.
	 */
	assert(!cpu_is_in_hardirq_handler());
	assert(!slist_is_empty(buf_list));
	assert(req->buf_list_done);

	sflags = softirq_save();
	if (req->status != -STATUS_IN_PROGRESS) {
		ret = -STATUS_FLUSHED;
	} else if (mcihost->current == req
			&& slist_is_empty(&mcihost->dmac_req.buf_list)
			&& mcihost->state == STATE_DATA_XFER) {
		slist_move_to_tail(&mcihost->dmac_req.buf_list, buf_list);
		dmac_chan_submit_request(mcihost->dmac_chan,
				&mcihost->dmac_req);
	} else {
		slist_move_to_tail(&req->buf_list, buf_list);
	}
	softirq_restore(sflags);

	return ret;
}

static void sdmmc_mcihost_dmac_done(struct dmac_channel *chan,
		struct dmac_request *req)
{
	struct sdmmc_mcihost	*mcihost = req->context;

	dbg_verbose("mcihost: dmac_done\n");

	atomic_set_bit(EVENT_DMA_COMPLETE, &mcihost->pending_events);
	softirq_raise(SOFTIRQ_ID_MCIHOST);
}

static void sdmmc_mcihost_chk_cmd(struct sdmmc_mcihost *mcihost,
		uint32_t status)
{
	struct sdmmc_request	*req = mcihost->current;
	uint32_t		error_flags;

	error_flags = MCI_RINDE | MCI_RDIRE | MCI_RENDE | MCI_RTOE | MCI_DTOE;
	if (req->cmd.flags & SDMMC_RSP_CRC)
		error_flags |= MCI_RCRCE;

	req->cmd.resp[0] = mci_readl(RSPR);
	req->cmd.resp[1] = mci_readl(RSPR);
	req->cmd.resp[2] = mci_readl(RSPR);
	req->cmd.resp[3] = mci_readl(RSPR);

	dbg_verbose("mcihost: RESP: %08x status = %08x\n", req->cmd.resp[0],
			status);

	if (status & error_flags) {
		if (status & (MCI_RTOE | MCI_DTOE)) {
			dbg_error("mcihost: CMD%u timeout. Flags: %08x "
					"Resp[0]: %08x\n", req->cmd.opcode,
					status,
					req->cmd.resp[0]);
			req->status = -STATUS_TIMEOUT;
		} else {
			dbg_error("mcihost: CMD%u error. Flags: %08x "
					"Resp[0]: %08x\n",req->cmd.opcode,
					status,
					req->cmd.resp[0]);
			req->status = -STATUS_IO_ERROR;
		}
		req->cmd.status = req->status;
	} else
		req->cmd.status = 0;
}

static void sdmmc_mcihost_chk_data(struct sdmmc_mcihost *mcihost,
		uint32_t status)
{
	struct sdmmc_request	*req = mcihost->current;
	uint32_t		error_flags;

	error_flags = MCI_DCRCE | MCI_DTOE;

	if (status & error_flags) {
		if (status & MCI_DTOE) {
			dbg_error("mcihost: Data timeout. Flags: %08x\n",
					status);
			if (!req->status)
				req->status = -STATUS_TIMEOUT;
		} else {
			dbg_error("mcihost: Data crc error. Flags: %08x\n",
					status);
			if (!req->status)
				req->status = -STATUS_IO_ERROR;
		}
	}
}
static void sdmmc_mcihost_chk_done(struct sdmmc_mcihost *mcihost,
		uint32_t status)
{
	struct sdmmc_request	*req = mcihost->current;
	uint32_t		error_flags;

	error_flags = MCI_RINDE | MCI_RDIRE | MCI_RENDE | MCI_RTOE |
			MCI_DCRCE | MCI_DTOE;

	if (status & error_flags) {
		if (status & (MCI_RTOE | MCI_DTOE)) {
			dbg_error("mcihost: End timeout. Flags: %08x\n",
					status);
			if (!req->status)
				req->status = -STATUS_TIMEOUT;
		} else {
			dbg_error("mcihost: End crc error. Flags: %08x\n",
					status);
			if (!req->status)
				req->status = -STATUS_IO_ERROR;
		}
	}
}

static void sdmmc_mcihost_check_status(struct sdmmc_mcihost *mcihost)
{
	struct sdmmc_slot	*slot;

	slot = mcihost->current->slot;
	mci_writel(ARGR, slot->card.rca);
	mci_writel(CMDR, MCI_CMDR_CMDNB(SDMMC_SEND_STATUS)
			| MCI_CMDR_RSPTYP_48BIT
			| MCI_CMDR_MAXLAT_64CYC);
	mci_writel(IER, MCI_CMDRDY);
}

static void sdmmc_mcihost_softirq(void *data)
{
	struct sdmmc_mcihost	*mcihost = data;
	struct sdmmc_request	*req;
	enum sdmmc_mcihost_state prev_state;
	uint32_t		status;

	/* Run the state machine as long as the state is changing */
	do {
		prev_state = mcihost->state;

		dbg_verbose("mcihost: softirq state %u events: %lx\n",
				mcihost->state, mcihost->pending_events);

		switch (mcihost->state) {
		case STATE_IDLE:
			break;

		case STATE_SENDING_CMD:
			if (!atomic_test_and_clear_bit(EVENT_CMD_COMPLETE,
						&mcihost->pending_events))
				break;

			dbg_verbose("    CMD done: status 0x%x\n",
					mcihost->cmd_status);

			sdmmc_mcihost_chk_cmd(mcihost, mcihost->cmd_status);

			req = mcihost->current;
			if (req->status != -STATUS_IN_PROGRESS) {
				/* Can't rely on XFRDONE for some reason */
				mcihost_reset_controller(mcihost);

				mcihost->state = STATE_IDLE;
				sdmmc_mcihost_req_close(mcihost);
				break;
			} else if (req->blocks == 0) {
				mci_writel(IER, MCI_XFRDONE);
				mcihost->state = STATE_FINISH_REQ;
				break;
			}

			mcihost->data_status = 0;
			if (test_bit(SDMMC_REQ_WRITE, &req->flags))
				mcihost->dmac_req.direction = DMA_TO_DEVICE;
			else
				mcihost->dmac_req.direction = DMA_FROM_DEVICE;

			if (!slist_is_empty(&req->buf_list)) {
				slist_move_to_tail(&mcihost->dmac_req.buf_list,
						&req->buf_list);
				dmac_chan_submit_request(mcihost->dmac_chan,
						&mcihost->dmac_req);
			}
			mcihost->state = prev_state = STATE_DATA_XFER;

			if (req->req_started)
				req->req_started(req);

			mci_writel(IER, MCI_DTOE);
			prev_state = mcihost->state = STATE_DATA_XFER;

			/* fall through */

		case STATE_DATA_XFER:
			if (atomic_test_and_clear_bit(EVENT_DATA_ERROR,
						&mcihost->pending_events)) {
				sdmmc_mcihost_chk_data(mcihost,
						mcihost->data_status);
				dmac_chan_reset(mcihost->dmac_chan);
				mcihost_reset_controller(mcihost);

				mcihost->state = STATE_IDLE;
				sdmmc_mcihost_req_close(mcihost);
				break;
			}

			if (atomic_test_and_clear_bit(EVENT_DMA_COMPLETE,
						&mcihost->pending_events))
				sdmmc_mcihost_load_data(mcihost);

			if (!atomic_test_and_clear_bit(EVENT_DATA_COMPLETE,
						&mcihost->pending_events)) {
				break;
			}

			dbg_verbose("    DATA done: status 0x%x\n",
					mcihost->data_status);

			req = mcihost->current;
			if (test_bit(SDMMC_REQ_STOP, &req->flags)) {
				sdmmc_mcihost_send_stop(mcihost,
						test_bit(SDMMC_REQ_WRITE,
							&req->flags));
				mcihost->state = STATE_SENDING_STOP;
			} else {
				mcihost->state = STATE_FINISH_REQ;
				mci_writel(IER, MCI_XFRDONE);
			}
			break;

		case STATE_SENDING_STOP:
			if (!atomic_test_and_clear_bit(EVENT_CMD_COMPLETE,
						&mcihost->pending_events))
				break;

			dbg_verbose("    STOP done: status 0x%x\n",
					mcihost->cmd_status);

			sdmmc_mcihost_chk_done(mcihost, mcihost->cmd_status);
			mci_writel(IER, MCI_XFRDONE);
			mcihost->state = prev_state = STATE_FINISH_REQ;

			/* fall through */

		case STATE_FINISH_REQ:
			if (!atomic_test_and_clear_bit(EVENT_REQ_COMPLETE,
						&mcihost->pending_events))
				break;

			dbg_verbose("    REQ done: status %d\n",
					mcihost->current->status);

			req = mcihost->current;
			if (!test_bit(SDMMC_REQ_WRITE, &req->flags)) {
				mcihost->state = STATE_IDLE;
				sdmmc_mcihost_req_close(mcihost);
				break;
			}

			/*
			 * Workaround for faulty busy detection: Poll
			 * the card status until the card is in TRAN
			 * state.
			 *
			 * This will be fixed in UC3A3 revE, but there
			 * are reports of buggy cards in out in the wild
			 * too...
			 */
			sdmmc_mcihost_check_status(mcihost);
			mcihost->state = prev_state = STATE_CARD_STATUS;

			/* fall through */

		case STATE_CARD_STATUS:
			if (!atomic_test_and_clear_bit(EVENT_CMD_COMPLETE,
						&mcihost->pending_events))
				break;

			if (mcihost->cmd_status & (MCI_RTOE | MCI_RENDE
						| MCI_RCRCE | MCI_RDIRE
						| MCI_RINDE)) {
				/* The card was probably removed. */
				/* TODO: Maybe flag this as an error? */
				mcihost->state = STATE_IDLE;
				sdmmc_mcihost_req_close(mcihost);
				break;
			}

			status = mci_readl(RSPR);
			dbg_verbose("    Card status: %08x\n", status);

			if (((status >> 9) & 0xf) != 4) {
				dbg_verbose("Bad card state: %08x\n", status);
				sdmmc_mcihost_check_status(mcihost);
			} else {
				mcihost->state = STATE_IDLE;
				sdmmc_mcihost_req_close(mcihost);
			}
			break;
		}
	} while (mcihost->state != prev_state);
}

static void sdmmc_mcihost_interrupt(void *data)
{
	struct sdmmc_mcihost	*mcihost = data;
	uint32_t		status;
	uint32_t		status_masked;

	status = mci_readl(SR);
	status_masked = status & mci_readl(IMR);

	dbg_verbose("mcihost interrupt: status=%08x (%08x)\n",
			status, status_masked);

	if (status_masked & MCI_DTOE) {
		mci_writel(IDR, MCI_DTOE);
		mcihost->data_status = status;
		set_bit(EVENT_DATA_ERROR, &mcihost->pending_events);
	}
	if (status_masked & MCI_CMDRDY) {
		mci_writel(IDR, MCI_CMDRDY);
		mcihost->cmd_status = status;
		set_bit(EVENT_CMD_COMPLETE, &mcihost->pending_events);
	}
	if (status_masked & MCI_FIFOEMPTY) {
		mci_writel(IDR, MCI_FIFOEMPTY);
		set_bit(EVENT_DATA_COMPLETE, &mcihost->pending_events);
	}
	if (status_masked & MCI_XFRDONE) {
		mci_writel(IDR, ~0UL);
		if (!mcihost->data_status)
			mcihost->data_status = status;
		dbg_verbose("xfrdone: status=0x%x\n", status);
		set_bit(EVENT_REQ_COMPLETE, &mcihost->pending_events);
	}

	softirq_raise(SOFTIRQ_ID_MCIHOST);
}
DEFINE_IRQ_HANDLER(mci, sdmmc_mcihost_interrupt, 0);

static struct sdmmc_slot *sdmmc_mcihost_get_slot(struct sdmmc_host *host,
		int slot_id)
{
	struct sdmmc_mcihost *mcihost = sdmmc_mcihost_of(host);

	assert(slot_id >= 0);
	assert(slot_id < SDMMC_SLOT_COUNT);

	return &mcihost->slots[slot_id].slot;
}

static bool sdmmc_mcihost_wp_is_active(struct sdmmc_host *host,
		struct sdmmc_slot *slot)
{
	assert(slot);

	switch (sdmmc_mcihost_get_hwslot(slot->id)) {
#if defined(CONFIG_SDMMC_MCIHOST_SLOT_A) && defined(BOARD_MCISLOT_A_WP)
	case SDMMC_MCIHOST_SLOT_A:
		return gpio_get_value(BOARD_MCISLOT_A_WP);
#endif
#if defined(CONFIG_SDMMC_MCIHOST_SLOT_B) && defined(BOARD_MCISLOT_B_WP)
	case SDMMC_MCIHOST_SLOT_B:
		return gpio_get_value(BOARD_MCISLOT_B_WP);
#endif
	default:
		/* If not wired up, assume write-enabled */
		return false;
	}
}

static void mcihost_dump_regs(void)
{
	unsigned int	offset;

	dbg_verbose("mcihost register dump:");
	for (offset = 0; offset < 0x100; offset += 4) {
		if (offset % 16 == 0)
			dbg_verbose("\n%04x:", offset);

		if (offset == MCI_RDR || offset == MCI_TDR)
			dbg_verbose(" xxxxxxxx");
		else
			dbg_verbose(" %08x",
				mmio_read32((void *)(MCI_BASE + offset)));
	}
	dbg_verbose("\n");
}

/*
 * This function is called from a workqueue once per second to check if any
 * requests have been ongoing for too long. If any such requests are found, a
 * nasty-looking message is displayed to the debug console (with lots of
 * detailed state information if verbose debugging is enabled), the MMC
 * controller and DMA controllers are reset and the request terminated.
 */
static void mcihost_watchdog_worker(void *data)
{
	struct sdmmc_mcihost	*mcihost = data;

	if (mcihost->state != STATE_IDLE
			&& timeout_has_expired(&mcihost->req_timeout)) {
		struct sdmmc_request	*req;

		req = mcihost->current;
		dbg_error("mcihost slot%u: timeout, state %u, status %d\n",
				req->slot->id, mcihost->state, req->status);
		dbg_verbose("  CMD%u(0x%x) flags 0x%lx status %d resp 0x%x\n",
				req->cmd.opcode, req->cmd.arg,
				req->cmd.flags, req->cmd.status,
				req->cmd.resp[0]);
		if (req->blocks)
			dbg_verbose(
				"  %s: %u blocks x %u, buf %p, xfered %zu\n",
					(req->flags & SDMMC_REQ_WRITE)
					? "WRITE" : "READ",
					req->blocks, req->block_size,
					slist_peek_head(&req->buf_list,
						struct buffer, node)->addr.ptr,
					req->bytes_xfered);
		if (req->flags & SDMMC_REQ_STOP)
			dbg_verbose("  STOP\n");

		mcihost_dump_regs();

		dmac_chan_reset(mcihost->dmac_chan);
		mcihost_reset_controller(mcihost);

		mcihost->state = STATE_IDLE;
		sdmmc_mcihost_req_close(mcihost);
	}

	delayed_work_run_us(&mcihost->watchdog_dw,
			&mcihost->watchdog_work,
			1000 * MCIHOST_WATCHDOG_PERIOD_MS);
}

static void sdmmc_mcihost_enable(struct sdmmc_host *host)
{
	struct sdmmc_mcihost	*mcihost = sdmmc_mcihost_of(host);
	int			i;

	board_mcislots_enable(0);

	i = 0;
#if defined(CONFIG_SDMMC_MCIHOST_SLOT_A) && defined(BOARD_MCISLOT_A_CD)
	sdmmc_cd_enable(mcihost->slots[i].slot.cd);
	i++;
#endif
#if defined(CONFIG_SDMMC_MCIHOST_SLOT_B) && defined(BOARD_MCISLOT_B_CD)
	sdmmc_cd_enable(mcihost->slots[i].slot.cd);
#endif
	mci_writel(CR, MCI_CR_MCIEN);

	/* Check for stalled requests every second */
	delayed_work_run_us(&mcihost->watchdog_dw,
			&mcihost->watchdog_work,
			1000 * MCIHOST_WATCHDOG_PERIOD_MS);
}

struct sdmmc_host *sdmmc_mcihost_init()
{
	struct sdmmc_mcihost *mcihost;
	int i;

	mcihost = malloc(sizeof(struct sdmmc_mcihost));
	if (mcihost == NULL) {
		dbg_printf("sdmmc_mcihost_init: Out of memory!\n");
		return NULL;
	}
	memset(mcihost, 0, sizeof(struct sdmmc_mcihost));
	mcihost->host.f_max = min(CONFIG_MAX_MMC_HZ,
			get_mci_pclk_rate(0) / 2);
	mcihost->host.f_min = (get_mci_pclk_rate(0) + 511) /  512;
	mcihost->host.ocr_avail = SDMMC_OCR_V_32_33 | SDMMC_OCR_V_33_34;
	slist_init(&mcihost->req_queue);
	mcihost->dmac_chan = dmac_mci_alloc_channel();
	if (mcihost->dmac_chan == NULL) {
		dbg_panic("sdmmc_mcihost_init: Failed to allocate DMA channel!\n");
		free(mcihost);
		return NULL;
	}
	dmac_req_init(&mcihost->dmac_req);
	mcihost->dmac_req.reg_width = DMAC_REG_WIDTH_32BIT;
	mcihost->dmac_req.req_done = sdmmc_mcihost_dmac_done;
	mcihost->dmac_req.context = mcihost;

	mcihost->host.enable = sdmmc_mcihost_enable;
	mcihost->host.slot_count = SDMMC_SLOT_COUNT;
	mcihost->host.get_slot = sdmmc_mcihost_get_slot;
	mcihost->host.power_up = sdmmc_mcihost_power_up;
	mcihost->host.power_down = sdmmc_mcihost_power_down;
	mcihost->host.set_bus_params = sdmmc_mcihost_set_bus_params;
	mcihost->host.set_voltage = sdmmc_mcihost_set_voltage;
	mcihost->host.wp_is_active = sdmmc_mcihost_wp_is_active;
	mcihost->host.submit_req = sdmmc_mcihost_submit_req;
	mcihost->host.submit_buf_list = sdmmc_mcihost_submit_buf_list;

	i = 0;
#ifdef CONFIG_SDMMC_MCIHOST_SLOT_A
	sdmmc_slot_init(&mcihost->slots[i].slot, &mcihost->host, i);
	mcihost->slots[i].sdc_reg = MCI_SDCSEL_SLOT_A;
#ifdef BOARD_MCISLOT_A_CD
	mcihost->slots[i].slot.cd = sdmmc_cd_init(&mcihost->slots[i].slot,
			BOARD_MCISLOT_A_CD);
#endif
	i++;
#endif /* CONFIG_SDMMC_MCIHOST_SLOT_A */

#ifdef CONFIG_SDMMC_MCIHOST_SLOT_B
	sdmmc_slot_init(&mcihost->slots[i].slot, &mcihost->host, i);
	mcihost->slots[i].sdc_reg = MCI_SDCSEL_SLOT_B;
#ifdef BOARD_MCISLOT_B_CD
	mcihost->slots[i].slot.cd = sdmmc_cd_init(&mcihost->slots[i].slot,
			BOARD_MCISLOT_B_CD);
#endif
#endif /* CONFIG_SDMMC_MCIHOST_SLOT_B */

	mci_writel(CR, MCI_CR_SWRST);
	mci_writel(SDCR, 0);

	softirq_set_handler(SOFTIRQ_ID_MCIHOST, sdmmc_mcihost_softirq, mcihost);
	setup_irq_handler(MCI_IRQ, mci, 0, mcihost);

	workqueue_init_item(&mcihost->watchdog_work, mcihost_watchdog_worker,
			mcihost);
	delayed_work_init(&mcihost->watchdog_dw, &mcihost_watchdog_timer,
			&mcihost_watchdog_workqueue);

	dbg_printf("sdmmc_host_init:  max/min %u/%u Hz\n",
			mcihost->host.f_max, mcihost->host.f_min);

	return &mcihost->host;
}
