/**
 * \file
 *
 * Copyright (C) 2009 Atmel Corporation. All rights reserved.
 *
 * \page License
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3. The name of Atmel may not be used to endorse or promote products derived
 * from this software without specific prior written permission.
 *
 * 4. This software may only be redistributed and used in connection with an
 * Atmel AVR product.
 *
 * THIS SOFTWARE IS PROVIDED BY ATMEL "AS IS" AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE
 * EXPRESSLY AND SPECIFICALLY DISCLAIMED. IN NO EVENT SHALL ATMEL BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 */
/*!
 * \file
 *
 * \brief USBB driver: Device part
 *
 * This file implements a USB Device Controller (UDC) driver utilizing
 * the USBB controller hardware.
 *
 * - Compiler:           IAR EWAVR32 and GNU GCC for AVR32
 * - Supported devices:  All AVR32 devices with a USBB module can be used.
 * - AppNote:
 *
 * \author               Atmel Corporation: http://www.atmel.com \n
 *                       Support and FAQ: http://support.atmel.no/
 *
 * \page License
 *
 * Copyright (C) 2008, Atmel Corporation All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3. The name of ATMEL may not be used to endorse or promote products derived
 * from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY ATMEL ``AS IS'' AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE EXPRESSLY AND
 * SPECIFICALLY DISCLAIMED. IN NO EVENT SHALL ATMEL BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
#include <assert.h>
#include <byteorder.h>
#include <debug.h>
#include <dma.h>
#include <io.h>
#include <softirq.h>
#include <status-codes.h>
#include <string.h>
#include <util.h>
#include <arch/interrupt.h>
#include <chip/memory-map.h>
#include <chip/usbb.h>
#include <usb/request.h>
#include <usb/function.h>
#include <usb/udc.h>

#include <app/config_usb.h>
#include <app/softirq.h>

#include "usbb_internal.h"
#include "usbb_regs.h"

/* Configuration sanity-checks */
#if defined(CONFIG_UDC_HIGH_SPEED)
# if !defined(CHIP_USBB_UDC_HIGH_SPEED)
#  error High-speed configuration not supported on this chip
# endif
# if !defined(CONFIG_UDC_FULL_SPEED)
#  error High-speed selected, but not full-speed
# endif
#endif
#if defined(CONFIG_UDC_FULL_SPEED) && !defined(CHIP_USBB_UDC_FULL_SPEED)
# error Full-speed configuration not supported on this chip
#endif
#if defined(CONFIG_UDC_LOW_SPEED)
# if !defined(CHIP_USBB_UDC_LOW_SPEED)
#  error Low-speed configuration not supported on this chip
# endif
# if defined(CONFIG_UDC_HIGH_SPEED) || defined(CONFIG_UDC_FULL_SPEED)
#  error Low-speed configuration must be selected exclusively
# endif
#endif

static inline struct usbb_udc *usbb_udc_of(struct udc *udc)
{
	return container_of(udc, struct usbb_udc, udc);
}

static void usbb_udc_req_done(struct udc *udc, struct usb_request *req,
		int status)
{
	req->status = status;

	if (req->req_done)
		req->req_done(udc, req);
}

static void usbb_udc_kill_last_in_bank(usb_ep_id_t ep_id)
{
	usbb_udc_write_reg(UECONSET(ep_id), USBB_UECON_KILLBK);
	while (usbb_udc_read_reg(UECON(ep_id)) & USBB_UECON_KILLBK)
		barrier();
}

static void usbb_udc_kill_first_out_bank(usb_ep_id_t ep_id)
{
	usbb_udc_write_reg(UESTACLR(ep_id), USBB_EP_RXOUTI);
	usbb_udc_write_reg(UECONCLR(ep_id), USBB_UECON_FIFOCON);
}

static void usbb_udc_kill_all_banks(usb_ep_id_t ep_id)
{
	bool	is_in = usbb_udc_read_reg(UECFG(ep_id)) & USBB_UECFG_EPDIR_IN;

	while (USBB_UESTA_GET_NBUSYBK(usbb_udc_read_reg(UESTA(ep_id)))) {
		if (is_in)
			usbb_udc_kill_last_in_bank(ep_id);
		else
			usbb_udc_kill_first_out_bank(ep_id);
	}
}

/**
 * \pre In interrupt handler and/or interrupts disabled.
 */
static void usbb_udc_dma_buf_done(struct buffer *buf)
{
	struct usbb_sw_dma_desc	*dma_desc = buf->dma_desc;

	usbb_dma_desc_free(dma_desc);
	buf->dma_desc = NULL;
}

static void usbb_udc_dma_req_done(struct udc *udc, struct slist *queue,
		struct usb_request *req, int status)
{
	slist_give_back_head(&req->buf_list, queue);

	usbb_udc_req_done(udc, req, status);
}

/**
 * \brief Submit an OUT request on the default control endpoint.
 *
 * This function queues a USB request for receiving OUT data on the
 * default control endpoint (ep0).
 *
 * \param udc The USB Device Controller instance
 * \param req Request to use for receiving the OUT data.
 *
 * \pre No other requests are queued on ep0.
 * \post ep0 has entered the data OUT phase.
 */
void udc_ep0_submit_out_req(struct udc *udc, struct usb_request *req)
{
	struct usbb_udc		*udcb = usbb_udc_of(udc);
	struct usbb_udc_ep	*ep = &udcb->ep[0];

	assert(!test_bit(USBB_EP_ACTIVE_XFER, &ep->flags));
	assert(slist_is_empty(&ep->buf_queue));
	assert(slist_is_empty(&ep->req_queue));
	assert(ep->buf_offset == 0);

	udcb->ctrl_state = EP0_STATE_DATA_OUT;
	set_bit(USBB_EP_ACTIVE_XFER, &ep->flags);
	slist_borrow_to_tail(&ep->buf_queue, &req->buf_list);
	slist_insert_tail(&ep->req_queue, &req->node);
	barrier();
	usbb_udc_write_reg(UECONSET(0), USBB_EP_RXOUTI);
}

/**
 * \brief Submit an IN request on the default control endpoint.
 *
 * This function queues a USB request for transmitting IN data on the
 * default control endpoint (ep0).
 *
 * \param udc The USB Device Controller instance
 * \param req Request containing IN data for transmission.
 *
 * \pre No other requests are queued on ep0.
 * \post ep0 has entered the data IN phase.
 */
void udc_ep0_submit_in_req(struct udc *udc, struct usb_request *req)
{
	struct usbb_udc		*udcb = usbb_udc_of(udc);
	struct usbb_udc_ep	*ep = &udcb->ep[0];

	assert(!test_bit(USBB_EP_ACTIVE_XFER, &ep->flags));
	assert(slist_is_empty(&ep->buf_queue));
	assert(slist_is_empty(&ep->req_queue));
	assert(ep->buf_offset == 0);

	udcb->ctrl_state = EP0_STATE_DATA_IN;
	set_bit(USBB_EP_ACTIVE_XFER, &ep->flags);
	slist_borrow_to_tail(&ep->buf_queue, &req->buf_list);
	slist_insert_tail(&ep->req_queue, &req->node);
	barrier();
	usbb_udc_write_reg(UECONSET(0), USBB_EP_TXINI);
}

/**
 * \brief Transmit IN data on the default control data synchronously.
 *
 * This function will submit IN data on the default control endpoint
 * (ep0) and busy-wait until it has been sent.
 *
 * \param udc The USB Device Controller instance.
 * \param data The data to be transmitted on ep0.
 * \param len The number of bytes to be transmitted.
 *
 * \return The number of bytes actually transmitted. This may be less
 *	than the requested number of bytes. When sending 8 bytes or
 *	less, it is safe to assume that everything will be sent.
 *
 * \note This function should only be used for small quantities of
 * data when it is impractical to submit a buffer asynchronously.
 *
 * \pre ep0 is ready to transmit data (no other buffers are queued.)
 * \post ep0 is ready to transmit data.
 */
int udc_ep0_write_sync(struct udc *udc, const void *data, size_t len)
{
	assert(!test_bit(USBB_EP_ACTIVE_XFER, &usbb_udc_of(udc)->ep[0].flags));
	assert(usbb_udc_read_reg(UESTA(0)) & USBB_EP_TXINI);

	len = min(len, CONFIG_USB_FUNC_MAXPACKETSIZE0);
	memcpy(usbb_ep_fifo(0), data, len);
	usbb_udc_write_reg(UESTACLR(0), USBB_EP_TXINI);

	while (!(usbb_udc_read_reg(UESTA(0)) & USBB_EP_TXINI))
		barrier();

	return len;
}

/**
 * \brief Send a status packet on the default control endpoint.
 *
 * This function will send a zero-length status packet on ep0. It does
 * not wait for completion, as a status packet marks the end of a
 * control transaction so no further action by the function driver is
 * necessary.
 *
 * \param udc The USB Device Controller instance.
 *
 * \pre ep0 is ready to transmit data (no other buffers are queued.)
 * \post ep0 has entered the status IN phase.
 */
void udc_ep0_send_status(struct udc *udc)
{
	struct usbb_udc	*udcb = usbb_udc_of(udc);

	dbg_printf("usbb-udc: send status\n");

	assert(!test_bit(USBB_EP_ACTIVE_XFER, &udcb->ep[0].flags));
	assert(usbb_udc_read_reg(UESTA(0)) & USBB_EP_TXINI);

	usbb_udc_write_reg(UESTACLR(0), USBB_EP_TXINI);
	usbb_udc_write_reg(UECONCLR(0), USBB_EP_RXOUTI);
	usbb_udc_write_reg(UECONSET(0), USBB_EP_TXINI);

	if (!usbb_udc_entering_test_mode(udcb))
		udcb->ctrl_state = EP0_STATE_STATUS_IN;
}

/**
 * \brief Signal that a status packet is expected on the default
 *	control endpoint.
 *
 * This function marks the end of the data IN phase, and signals that
 * a status OUT packet is expected.
 *
 * \param udc The USB Device Controller instance.
 *
 * \pre ep0 is ready to transmit data (no other buffers are queued.)
 * \post ep0 has entered the status OUT phase.
 */
void udc_ep0_expect_status(struct udc *udc)
{
	struct usbb_udc	*udcb = usbb_udc_of(udc);

	assert(!test_bit(USBB_EP_ACTIVE_XFER, &udcb->ep[0].flags));
	assert(usbb_udc_read_reg(UESTA(0)) & USBB_EP_TXINI);

	/*
	 * Don't bother waiting for STATUS OUT. The RX interrupt will
	 * get cleared when we receive the next SETUP packet.
	 */
	udcb->ctrl_state = EP0_STATE_STATUS_OUT;
}

/**
 * \brief Signal that the UDC is to change its USB address after the
 *	status IN phase is complete.
 *
 * \param udc The USB Device Controller instance.
 * \param addr The new USB address to be used starting from the next
 *	control transaction.
 */
void udc_set_address(struct udc *udc, unsigned int addr)
{
	/* We'll just peek at the SETUP request when status is done. */
}

/**
 * \internal
 * \brief Submit queued buffers on a non-control OUT endpoint.
 *
 * \note This function will unmask interrupts while processing the
 * queue, but will return with interrupts masked.
 *
 * \pre ep->active == NULL
 * \pre Interrupts masked
 */
static void usbb_udc_submit_out_queue(struct usbb_udc *udcb,
		usb_ep_id_t ep_id, struct usbb_udc_ep *ep)
{
	struct usbb_sw_dma_desc	*dma;
	struct usbb_sw_dma_desc	*dma_next;
	struct buffer		*buf;
	struct buffer		*next;
	struct usb_request	*req;
	phys_addr_t		phys;
	uint32_t		ctrl;

	dbg_printf("ep%u-out: submit buf_queue %p req_queue %p status=%08x\n",
			ep_id, slist_peek_head_node(&ep->buf_queue),
			slist_peek_head_node(&ep->req_queue),
			usbb_udc_dma_read_reg(ep_id, STATUS));

	assert(!test_bit(USBB_EP_ACTIVE_XFER, &ep->flags));

	if (slist_is_empty(&ep->buf_queue)) {
		assert(slist_is_empty(&ep->req_queue));
		goto queue_empty;
	}

	set_bit(USBB_EP_ACTIVE_XFER, &ep->flags);
	cpu_irq_enable();

	dma = usbb_dma_desc_alloc(&phys);
	if (!dma) {
		cpu_irq_disable();
		goto no_desc;
	}

	usbb_udc_dma_write_reg(ep_id, NEXTDESC, phys);

	cpu_irq_disable();
	if (!test_bit(USBB_EP_ENABLED, &ep->flags))
		goto disabled;

	req = slist_peek_head(&ep->req_queue, struct usb_request, node);
	slist_for_each_safe(&ep->buf_queue, buf, next, node) {
		if (!dma)
			break;

		dma->phys = phys;
		dma_next = NULL;
		if (!slist_node_is_last(&req->buf_list, &buf->node))
			dma_next = usbb_dma_desc_alloc(&phys);

		/* REVISIT: Is BUFF_CLOSE/EOT_IRQ always desirable? */
		ctrl = USBB_DMA_CH_EN | USBB_DMA_BUFF_CLOSE_IN_EN
			/* | USBB_DMA_DMAEND_EN */
			| USBB_DMA_BYTE_LEN(buf->len);

		if (req->buf_list.last == &buf->node) {
			if (req->req_done)
				ctrl |= USBB_DMA_EOBUFF | USBB_DMA_EOT;
			req = slist_peek_next(&req->node,
					struct usb_request, node);
		}

		dma->hw.buf_addr = buf->addr.phys;

		cpu_irq_enable();

		/*
		 * If descriptor pool is exhausted, or this is the
		 * last buffer, we need an interrupt to advance the
		 * queue.
		 */
		if (!dma_next) {
			ctrl |= USBB_DMA_EOBUFF | USBB_DMA_EOT;
		} else {
			ctrl |= USBB_DMA_LD_NXT_CH_DESC_EN;
			dma->hw.next = phys;
		}

		dma->hw.control = ctrl;

		dbg_printf("OUT %08lx: %08x %08x %08x\n",
				dma->phys, dma->hw.next,
				dma->hw.buf_addr, dma->hw.control);

		cpu_irq_disable();
		if (!test_bit(USBB_EP_ENABLED, &ep->flags))
			goto disabled_free_next;

		buf->dma_desc = dma;
		dma = dma_next;
	}

	/* The compiler must not move any stores beyond this point */
	barrier();
	usbb_udc_dma_write_reg(ep_id, CONTROL, USBB_DMA_LD_NXT_CH_DESC_EN);
	usbb_udc_write_reg(UDINTESET, USBB_UDINT_DMA(ep_id));

	return;

disabled_free_next:
	usbb_dma_desc_free(dma_next);
disabled:
	usbb_dma_desc_free(dma);
no_desc:
	clear_bit(USBB_EP_ACTIVE_XFER, &ep->flags);
queue_empty:
	usbb_udc_write_reg(UDINTECLR, USBB_UDINT_DMA(ep_id));
}

/**
 * \internal
 * \brief Submit queued buffers on a non-control IN endpoint.
 *
 * \note This function will unmask interrupts while processing the
 * queue, but will return with interrupts masked.
 *
 * \pre ep->active == NULL
 * \pre Interrupts masked
 */
static void usbb_udc_submit_in_queue(struct usbb_udc *udcb,
		usb_ep_id_t ep_id, struct usbb_udc_ep *ep)
{
	struct usbb_sw_dma_desc	*dma;
	struct usbb_sw_dma_desc	*dma_next;
	struct buffer		*buf;
	struct buffer		*next;
	struct usb_request	*req;
	phys_addr_t		phys;
	uint32_t		ctrl;

	dbg_printf("ep%u-in: submit buf_queue %p req_queue %p status=%08x\n",
			ep_id, slist_peek_head_node(&ep->buf_queue),
			slist_peek_head_node(&ep->req_queue),
			usbb_udc_dma_read_reg(ep_id, STATUS));

	assert(!test_bit(USBB_EP_ACTIVE_XFER, &ep->flags));

	if (slist_is_empty(&ep->buf_queue)) {
		assert(slist_is_empty(&ep->req_queue));
		goto queue_empty;
	}

	set_bit(USBB_EP_ACTIVE_XFER, &ep->flags);
	cpu_irq_enable();

	dma = usbb_dma_desc_alloc(&phys);
	if (!dma) {
		cpu_irq_disable();
		goto no_desc;
	}

	usbb_udc_dma_write_reg(ep_id, NEXTDESC, phys);

	cpu_irq_disable();
	if (!test_bit(USBB_EP_ENABLED, &ep->flags))
		goto disabled;

	req = slist_peek_head(&ep->req_queue, struct usb_request, node);
	slist_for_each_safe(&ep->buf_queue, buf, next, node) {
		if (!dma)
			break;

		dma->phys = phys;
		dma_next = NULL;
		if (!slist_node_is_last(&req->buf_list, &buf->node))
			dma_next = usbb_dma_desc_alloc(&phys);

		ctrl = USBB_DMA_CH_EN | USBB_DMA_BYTE_LEN(buf->len);

		/*
		 * Terminate the transfer with a short packet if
		 * requested, or if the buffer does not end on an
		 * endpoint boundary and it's not continued.
		 */
		if (req->buf_list.last == &buf->node) {
			if (test_bit(USB_REQ_SHORT_PKT, &req->flags)
					|| (buf->len % ep->maxpacket))
				ctrl |= USBB_DMA_DMAEND_EN;
			if (req->req_done)
				ctrl |= USBB_DMA_EOBUFF;
			req = slist_peek_next(&req->node,
					struct usb_request, node);
		}

		dma->hw.buf_addr = buf->addr.phys;

		cpu_irq_enable();

		/*
		 * If descriptor pool is exhausted, or this is the
		 * last buffer, we need an interrupt to advance the
		 * queue.
		 */
		if (!dma_next) {
			ctrl |= USBB_DMA_EOBUFF;
		} else {
			ctrl |= USBB_DMA_LD_NXT_CH_DESC_EN;
			dma->hw.next = phys;
		}

		dma->hw.control = ctrl;

		dbg_printf("IN  %08lx: %08x %08x %08x\n",
				dma->phys, dma->hw.next,
				dma->hw.buf_addr, dma->hw.control);

		cpu_irq_disable();
		if (!test_bit(USBB_EP_ENABLED, &ep->flags))
			goto disabled_free_next;

		buf->dma_desc = dma;
		dma = dma_next;
	}

	/* The compiler must not move any stores beyond this point */
	barrier();
	usbb_udc_dma_write_reg(ep_id, CONTROL, USBB_DMA_LD_NXT_CH_DESC_EN);
	usbb_udc_write_reg(UDINTESET, USBB_UDINT_DMA(ep_id));

	return;

disabled_free_next:
	usbb_dma_desc_free(dma_next);
disabled:
	usbb_dma_desc_free(dma);
no_desc:
	clear_bit(USBB_EP_ACTIVE_XFER, &ep->flags);
queue_empty:
	usbb_udc_write_reg(UDINTECLR, USBB_UDINT_DMA(ep_id));
}

static void usbb_udc_verify_req(struct usb_request *req)
{
#ifdef DEBUG
	struct buffer	*buf;

	assert(!slist_is_empty(&req->buf_list));
	usb_req_for_each_buffer(req, buf) {
		dbg_printf("verifying buf %p: %p %p %08lx %zu\n",
				buf, buf->dma_desc, buf->addr.ptr,
				buf->addr.phys, buf->len);
		assert(!buf->dma_desc);
	}
#endif
}

/**
 * \brief Submit an OUT request on a non-control endpoint.
 *
 * This function queues a USB request for receiving OUT data on a
 * non-control endpoint.
 *
 * \param udc The USB Device Controller instance
 * \param ep_id The endpoint ID on which to queue the buffer.
 * \param req Request containing OUT data for reception.
 *
 * \pre Interrupts not masked
 * \pre ep > 0 && ep < #CONFIG_USBB_NR_EP
 */
void udc_ep_submit_out_req(struct udc *udc, usb_ep_id_t ep_id,
		struct usb_request *req)
{
	struct usbb_udc		*udcb = usbb_udc_of(udc);
	struct usbb_udc_ep	*ep = &udcb->ep[ep_id];
	bool			queued = true;

	assert(cpu_irq_is_enabled());
	assert(ep_id > 0 && ep_id < CONFIG_USBB_NR_EP);
	usbb_udc_verify_req(req);

	cpu_irq_disable();
	if (test_bit(USBB_EP_ENABLED, &ep->flags)) {
		slist_insert_tail(&ep->req_queue, &req->node);
		slist_borrow_to_tail(&ep->buf_queue, &req->buf_list);
		if (!test_bit(USBB_EP_ACTIVE_XFER, &ep->flags))
			usbb_udc_submit_out_queue(udcb, ep_id, ep);
	} else {
		queued = false;
	}
	cpu_irq_enable();

	if (!queued)
		usbb_udc_req_done(udc, req, -STATUS_IO_ERROR);
}

/**
 * \brief Submit an IN request on a non-control endpoint.
 *
 * This function queues a USB request for transmitting IN data on a
 * non-control endpoint.
 *
 * \param udc The USB Device Controller instance
 * \param ep_id The endpoint ID on which to queue the buffer.
 * \param req Request containing IN data for transmission.
 *
 * \pre Interrupts not masked
 * \pre ep > 0 && ep < #CONFIG_USBB_NR_EP
 */
void udc_ep_submit_in_req(struct udc *udc, usb_ep_id_t ep_id,
		struct usb_request *req)
{
	struct usbb_udc		*udcb = usbb_udc_of(udc);
	struct usbb_udc_ep	*ep = &udcb->ep[ep_id];
	bool			queued = true;

	assert(cpu_irq_is_enabled());
	assert(ep_id > 0 && ep_id < CONFIG_USBB_NR_EP);
	usbb_udc_verify_req(req);

	cpu_irq_disable();
	if (test_bit(USBB_EP_ENABLED, &ep->flags)) {
		slist_insert_tail(&ep->req_queue, &req->node);
		slist_borrow_to_tail(&ep->buf_queue, &req->buf_list);
		if (!test_bit(USBB_EP_ACTIVE_XFER, &ep->flags))
			usbb_udc_submit_in_queue(udcb, ep_id, ep);
	} else {
		queued = false;
	}
	cpu_irq_enable();

	if (!queued)
		usbb_udc_req_done(udc, req, -STATUS_IO_ERROR);
}

/**
 * \brief Check if a given endpoint is halted.
 *
 * \param udc USB Device Controller instance.
 * \param ep The ID of the endpoint to check.
 *
 * \retval 1 if \a ep is halted, i.e. the controller will respond
 *	with a STALL handshake to any transaction other than SETUP.
 * \retval 0 if \a ep is not halted.
 * \retval -1 if the endpoint address is invalid.
 */
int udc_ep_is_halted(struct udc *udc, usb_ep_id_t ep)
{
	if (ep >= CONFIG_USBB_NR_EP)
		return -1;

	return !!(usbb_udc_read_reg(UECON(ep)) & USBB_UECON_STALLRQ);
}

/**
 * \brief Set the halted state of an endpoint.
 *
 * After this function is called, any transaction on \a ep will result
 * in a STALL hanshake being sent. Any pending transactions will be performed
 * first, however.
 *
 * \param udc USB Device Controller instance.
 * \param ep The ID of the endpoint to be halted.
 *
 * \retval 0 if the endpoint was successfully halted.
 * \retval -1 if the endpoint address is invalid.
 */
int udc_ep_set_halt(struct udc *udc, usb_ep_id_t ep)
{
	uint32_t	uecfg;

	dbg_printf("usbb-udc: ep%d: set halt\n", ep);

	if (ep >= CONFIG_USBB_NR_EP)
		return -1;

	/*
	 * Even though the function driver takes care not to request
	 * stall until it has received a callback for the data
	 * transfer, there's still a chance that the data transfer may
	 * get STALLed.
	 *
	 * This is because the callback happens when the DMA transfer
	 * is complete, not when the transfers is actually done. At
	 * this point, there may still be IN data stuck in a bank
	 * waiting for the host to request it, and if so, the host
	 * will see a STALL instead of the data it asked for.
	 *
	 * Work around this by waiting for all IN banks to become
	 * empty before requesting a STALL. Ideally, an interrupt
	 * should be used, but that may cause us to send additional
	 * data before setting the STALL request, which would be just
	 * as bad.
	 *
	 * One way to improve this would be to set a flag indicating
	 * that the endpoint is really stalled and refusing to submit
	 * any more DMA requests until the endpoint has been
	 * un-stalled. When NBUSYBK becomes 0, we can set STALLRQ and
	 * restart the queue when the stall is cleared by the host or
	 * some driver.
	 */
	uecfg = usbb_udc_read_reg(UECFG(ep));
	if (uecfg & USBB_UECFG_EPDIR_IN) {
		uint32_t	uesta;

		/* Request stall as soon as the FIFO is empty */
		do {
			uesta = usbb_udc_read_reg(UESTA(ep));
		} while (USBB_UESTA_GET_NBUSYBK(uesta) != 0);
	}

	usbb_udc_write_reg(UECONSET(ep), USBB_UECON_STALLRQ);

	if (ep != 0 && !(uecfg & USBB_UECFG_EPDIR_IN)) {
		/*
		 * Flush the FIFO for OUT endpoints. The caller may be
		 * stalling because it doesn't want any more data, but
		 * the controller may have already received some...
		 */
		usbb_udc_kill_all_banks(ep);
	}

	return 0;
}

/**
 * \brief Clear the halted state of an endpoint.
 *
 * After this function is called, any transaction on \a ep will
 * be handled normally, i.e. a STALL hanshake will not be sent, and
 * the data toggle sequence will start at DATA0.
 *
 * \param udc USB Device Controller instance.
 * \param ep The ID of the endpoint to be un-halted.
 *
 * \retval 0 if the endpoint was successfully halted.
 * \retval -1 if the endpoint address is invalid.
 */
int udc_ep_clear_halt(struct udc *udc, usb_ep_id_t ep)
{
	struct usbb_udc	*udcb = usbb_udc_of(udc);

	dbg_printf("usbb-udc: ep%d: clear halt (%swedged)\n", ep,
			test_bit(USBB_EP_WEDGE, &udcb->ep[ep].flags)
			? "" : "not ");

	if (ep >= CONFIG_USBB_NR_EP)
		return -1;

	/* Always reset data toggle sequence */
	usbb_udc_write_reg(UECONSET(ep), USBB_UECON_RSTDT);

	if (!test_bit(USBB_EP_WEDGE, &udcb->ep[ep].flags)) {
		/* Clear the STALL request */
		usbb_udc_write_reg(UECONCLR(ep), USBB_UECON_STALLRQ);
	}

	return 0;
}

/**
 * \brief Check if a given endpoint is wedged.
 *
 * A wedged endpoint is a halted endpoint where udc_ep_clear_halt()
 * requests are ignored. To un-halt an wedged endpoint, first call
 * udc_ep_clear_wedge(), then call udc_ep_clear_halt().
 *
 * \param udc USB Device Controller instance.
 * \param ep The ID of the endpoint to check.
 *
 * \retval true if \a ep is wedged
 * \retval false if \a ep is not wedged.
 *
 * \pre ep < CONFIG_USBB_NR_EP
 */
bool udc_ep_is_wedged(struct udc *udc, usb_ep_id_t ep)
{
	struct usbb_udc	*udcb = usbb_udc_of(udc);

	assert(ep < CONFIG_USBB_NR_EP);

	return test_bit(USBB_EP_WEDGE, &udcb->ep[ep].flags);
}
/**
 * \brief Set the wedged state of an endpoint.
 *
 * After this function is called, any transaction on \a ep will result
 * in a STALL hanshake being sent, and all requests to clear the halt
 * condition will be ignored. Any pending transactions will be
 * performed first, however.
 *
 * \param udc USB Device Controller instance.
 * \param ep The ID of the endpoint to be wedged.
 *
 * \pre ep < CONFIG_USBB_NR_EP
 */
void udc_ep_set_wedge(struct udc *udc, usb_ep_id_t ep)
{
	struct usbb_udc	*udcb = usbb_udc_of(udc);
	unsigned long	iflags;
	int		ret;

	dbg_printf("usbb-udc: ep%d: set wedge\n", ep);

	iflags = cpu_irq_save();
	set_bit(USBB_EP_WEDGE, &udcb->ep[ep].flags);

	/*
	 * This function isn't called in response to host control
	 * requests, so it's always a bug when ep isn't valid.
	 */
	ret = udc_ep_set_halt(udc, ep);
	assert(!ret);
	cpu_irq_restore(iflags);
}

/**
 * \brief Clear the wedged state of an endpoint.
 *
 * After this function is called, the endpoint halt condition may be
 * cleared by calling udc_ep_clear_halt(). In particular, the host is
 * allowed to clear the halt condition using the ClearFeature(HALT)
 * control request.
 *
 * This function may be called even if the endpoint isn't wedged, but
 * if it is wedged, it must be halted too.
 *
 * \param udc USB Device Controller instance.
 * \param ep The ID of the endpoint to be un-wedged.
 *
 * \pre ep < CONFIG_USBB_NR_EP
 */
void udc_ep_clear_wedge(struct udc *udc, usb_ep_id_t ep)
{
	struct usbb_udc	*udcb = usbb_udc_of(udc);
	unsigned long	iflags;

	dbg_printf("usbb-udc: ep%d: clear wedge\n", ep);

	assert(ep < CONFIG_USBB_NR_EP);

	iflags = cpu_irq_save();

	assert(!test_bit(USBB_EP_WEDGE, &udcb->ep[ep].flags)
			|| udc_ep_is_halted(udc, ep) > 0);

	clear_bit(USBB_EP_WEDGE, &udcb->ep[ep].flags);
	cpu_irq_restore(iflags);
}

/**
 * Configure an endpoint at the hardware level.
 * \param id Endpoint number.
 * \param size Maximum packet size.
 * \param type Endpoint transfer type.
 * \param is_in true if the endpoint is an IN endpoint. Must be false
 *	for control endpoints.
 * \param nr_banks Number of FIFO banks (1, 2 or 3).
 * \return STATUS_OK if the endpoint was configured successfully or a
 *	negative status_code otherwise.
 */
static int usbb_udc_configure_ep(unsigned int id, unsigned int size,
		enum usb_ep_xfer_type type, bool is_in, unsigned int nr_banks,
		bool autosw)
{
	unsigned long	iflags;
	uint32_t	config;
	uint32_t	uerst;

	assert(nr_banks >= 1 && nr_banks <= 3 && size >= 8);

	config = (USBB_UECFG_ALLOC
			| USBB_UECFG_EPBK(nr_banks - 1)
			| USBB_UECFG_EPSIZE(ilog2(size / 8))
			| USBB_UECFG_EPTYPE(type));
	if (is_in)
		config |= USBB_UECFG_EPDIR_IN;

	usbb_udc_write_reg(UECFG(id), config);
	if (!(usbb_udc_read_reg(UESTA(id)) & USBB_UESTA_CFGOK)) {
		dbg_printf("ep%u: Configuration %08x invalid\n", id, config);
		return -STATUS_INVALID_PARAM;
	}

	iflags = cpu_irq_save();
	uerst = usbb_udc_read_reg(UERST);
	usbb_udc_write_reg(UERST, uerst | USBB_UERST_EPRST(id));
	usbb_udc_write_reg(UERST, uerst | USBB_UERST_EPEN(id));
	cpu_irq_restore(iflags);

	if (autosw)
		usbb_udc_write_reg(UECFG(id), config | USBB_UECFG_EPAUTOSW);

	return STATUS_OK;
}

static void usbb_udc_ep_flush(struct usbb_udc *udcb, usb_ep_id_t ep_id,
		struct usbb_udc_ep *ep)
{
	struct udc		*udc = &udcb->udc;
	struct buffer		*buf;
	struct buffer		*next_buf;
	struct usb_request	*req;
	unsigned long		iflags;

	assert(ep_id != 0 && ep_id < CONFIG_USBB_NR_EP);
	assert(test_bit(USBB_EP_ENABLED, &ep->flags));

	/*
	 * First, reset the hardware state, but don't disable the
	 * endpoint or reset any data toggles.
	 */
	iflags = cpu_irq_save();

	/* Prevent queueing new requests */
	clear_bit(USBB_EP_ENABLED, &ep->flags);
	clear_bit(USBB_EP_ACTIVE_XFER, &ep->flags);

	/* Disable EP and DMA interrupts */
	usbb_udc_write_reg(UDINTECLR, USBB_UDINT_EP(ep_id)
			| USBB_UDINT_DMA(ep_id));

	/*
	 * Make sure there's no pending DMA descriptor load and clear
	 * DMA interrupts
	 */
	usbb_udc_dma_write_reg(ep_id, CONTROL, 0);
	usbb_udc_dma_read_reg(ep_id, STATUS);

	/* Flush all data from the FIFO */
	usbb_udc_kill_all_banks(ep_id);
	cpu_irq_restore(iflags);

	/* Then, terminate all queued requests */
	if (!slist_is_empty(&ep->req_queue)) {
		req = slist_pop_head(&ep->req_queue, struct usb_request, node);

		slist_for_each_safe(&ep->buf_queue, buf, next_buf, node) {
			usbb_udc_dma_buf_done(buf);
			dbg_printf("buf %p req [%p %p]\n",
					&buf->node, req->buf_list.first.next,
					req->buf_list.last);
			if (req->buf_list.last == &buf->node) {
				/* REVISIT: introduce STATUS_SHUTDOWN? */
				usbb_udc_dma_req_done(udc, &ep->buf_queue,
						req, -STATUS_IO_ERROR);
				if (!slist_is_empty(&ep->req_queue))
					req = slist_pop_head(&ep->req_queue,
							struct usb_request,
							node);
				else
					assert(slist_is_empty(&ep->buf_queue));
			}
		}

		assert(slist_is_empty(&ep->buf_queue));
		assert(slist_is_empty(&ep->req_queue));
	} else {
		assert(slist_is_empty(&ep->buf_queue));
	}
}

/**
 * \brief Terminate all pending requests on an endpoint.
 *
 * This function will flush an endpoint, terminating all queued
 * requests with an error status. After this function returns, the
 * endpoint request queue will be empty.
 *
 * \param udc USB Device Controller instance.
 * \param ep_id USB Endpoint ID previously returned by udc_ep_create().
 */
void udc_ep_flush(struct udc *udc, usb_ep_id_t ep_id)
{
	struct usbb_udc		*udcb = usbb_udc_of(udc);
	struct usbb_udc_ep	*ep;
	unsigned long		iflags;

	dbg_printf("usbb-udc: flush ep%u\n", ep_id);

	ep = &udcb->ep[ep_id];
	usbb_udc_ep_flush(udcb, ep_id, ep);

	/* Allow queueing new requests */
	iflags = cpu_irq_save();
	set_bit(USBB_EP_ENABLED, &ep->flags);
	cpu_irq_restore(iflags);
}

/**
 * \brief Create a new endpoint.
 *
 * Create a new endpoint matching a given endpoint descriptor. The
 * transfer type, endpoint address, and FIFO bank size parameters are
 * taken from the descriptor.
 *
 * \param udc USB Device Controller instance.
 * \param desc USB Endpoint Descriptor for the new endpoint.
 * \param nr_banks The number of FIFO banks to allocate.
 *
 * \return A cookie identifying the new endpoint, or a negative error
 *	code.
 */
usb_ep_id_t udc_ep_create(struct udc *udc,
		const struct usb_endpoint_descriptor *desc,
		unsigned int nr_banks)
{
	struct usbb_udc		*udcb = usbb_udc_of(udc);
	struct usbb_udc_ep	*ep;
	unsigned int		index;
	unsigned long		iflags;
	int			ret;

	index = usb_ep_index(desc);
	assert(index > 0 && index < CONFIG_USBB_NR_EP);

	dbg_printf("usbb-udc: create ep%u addr: %02x attr: %02x banks: %u\n",
			index, desc->bEndpointAddress, desc->bmAttributes,
			nr_banks);

	ret = 0;
	ep = &udcb->ep[index];

	iflags = cpu_irq_save();
	if (ep->desc) {
		/* REVISIT: Perhaps introduce STATUS_BUSY? */
		ret = -STATUS_INVALID_PARAM;
	} else {
		ep->desc = desc;
		ep->flags = 0;
	}
	cpu_irq_restore(iflags);
	if (ret < 0) {
		dbg_printf("usbb-udc: ep%u is busy\n", index);
		return ret;
	}

	ret = usbb_udc_configure_ep(index, le16_to_cpu(desc->wMaxPacketSize),
				usb_ep_xfer(desc), usb_ep_is_in(desc),
				nr_banks, true);
	if (ret < 0) {
		ep->desc = NULL;
		return ret;
	}

	usbb_udc_write_reg(UDINTESET, USBB_UDINT_EP(index));

	slist_init(&ep->req_queue);
	slist_init(&ep->buf_queue);

	ep->maxpacket = le16_to_cpu(desc->wMaxPacketSize);

	set_bit(USBB_EP_ENABLED, &ep->flags);

	return index;
}

/**
 * \brief Destroy a previously created endpoint.
 *
 * This function will disable the specified endpoint, terminating all
 * queued buffers.
 *
 * \param udc USB Device Controller instance.
 * \param ep_id USB Endpoint ID previously returned by udc_ep_create().
 */
void udc_ep_destroy(struct udc *udc, usb_ep_id_t ep_id)
{
	struct usbb_udc		*udcb = usbb_udc_of(udc);
	struct usbb_udc_ep	*ep;
	uint32_t		uerst;
	unsigned long		iflags;

	dbg_printf("usbb-udc: destroy ep%u\n", ep_id);

	assert(ep_id != 0 && ep_id < CONFIG_USBB_NR_EP);

	ep = &udcb->ep[ep_id];
	usbb_udc_ep_flush(udcb, ep_id, ep);

	/* Now that we've flushed the queue, disable the endpoint */
	iflags = cpu_irq_save();
	uerst = usbb_udc_read_reg(UERST);
	usbb_udc_write_reg(UERST, uerst & ~USBB_UERST_EPEN(ep_id));
	usbb_udc_write_reg(UECFG(ep_id), 0);

	/* Allow re-use after all the cleanup has been done */
	ep->flags = 0;
	ep->desc = NULL;
	cpu_irq_restore(iflags);
}

static void usbb_udc_do_test_mode(struct usbb_udc *udcb)
{
	static const char test_packet_data[] = {
		/* JKJKJKJK * 9 */
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
		/* JJKKJJKK * 8 */
		0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,
		/* JJKKJJKK * 8 */
		0xEE, 0xEE, 0xEE, 0xEE, 0xEE, 0xEE, 0xEE, 0xEE,
		/* JJJJJJJKKKKKKK * 8 */
		0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
		0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
		/* JJJJJJJK * 8 */
		0x7F, 0xBF, 0xDF, 0xEF, 0xF7, 0xFB, 0xFD,
		/* {JKKKKKKK * 10}, JK */
		0xFC, 0x7E, 0xBF, 0xDF, 0xEF, 0xF7, 0xFB, 0xFD, 0x7E
	};
	uint32_t	udcon;
	int		ret;

	/* Disable all interrupts */
	usbb_udc_write_reg(UDINTECLR, ~0UL);
	usbb_udc_read_reg(UDINTE);

	/*
	 * Disable all endpoints except ep0. They should have been
	 * disabled already since udc_lib reset the configuration.
	 */
	usbb_udc_write_reg(UERST, USBB_UERST_EPEN(0));
	usbb_udc_read_reg(UERST);

	udcon = usbb_udc_read_reg(UDCON);

	switch (udcb->ctrl_state) {
	case EP0_STATE_TEST_J:
		udcon |= USBB_UDCON_TSTJ;
		usbb_udc_write_reg(UDCON, udcon);
		dbg_info("usbb_udc: Entered Test_J mode\n");
		break;

	case EP0_STATE_TEST_K:
		udcon |= USBB_UDCON_TSTK;
		usbb_udc_write_reg(UDCON, udcon);
		dbg_info("usbb_udc: Entered Test_K mode\n");
		break;

	case EP0_STATE_TEST_SE0_NAK:
		/* Disable and de-allocate ep0 (all other endpoints are
		 * already disabled)
		 */
		usbb_udc_write_reg(UERST, 0);
		usbb_udc_write_reg(UECFG(0), 0);

		/* Reconfigure ep0 for SE0_NAK */
		ret = usbb_udc_configure_ep(0, 64, USB_EP_XFER_BULK,
				true, 1, false);
		if (!ret)
			dbg_info("usbb_udc: Entered Test_SE0_NAK mode\n");
		break;

	case EP0_STATE_TEST_PACKET:
		/* Disable and de-allocate ep0 (all other endpoints are
		 * already disabled)
		 */
		usbb_udc_write_reg(UERST, 0);
		usbb_udc_write_reg(UECFG(0), 0);

		/* Reconfigure ep0 for TEST_PACKET */
		ret = usbb_udc_configure_ep(0, 64, USB_EP_XFER_BULK,
				true, 1, false);
		if (!ret) {
			udcon |= USBB_UDCON_TSTPCKT;
			usbb_udc_write_reg(UDCON, udcon);
			memcpy(usbb_ep_fifo(0), test_packet_data,
					sizeof(test_packet_data));
			usbb_udc_write_reg(UECONCLR(0), USBB_UECON_FIFOCON);
			dbg_info("usbb_udc: Entered Test_Packet mode\n");
		}
		break;

	default:
		dbg_error("usbb_udc: Invalid test state %u!\n",
				udcb->ctrl_state);
		break;
	}
}

int udc_enter_test_mode(struct udc *udc, unsigned int mode)
{
	struct usbb_udc		*udcb = usbb_udc_of(udc);

	switch (mode) {
	case USB_TEST_J:
		udcb->ctrl_state = EP0_STATE_TEST_J;
		break;
	case USB_TEST_K:
		udcb->ctrl_state = EP0_STATE_TEST_K;
		break;
	case USB_TEST_SE0_NAK:
		udcb->ctrl_state = EP0_STATE_TEST_SE0_NAK;
		break;
	case USB_TEST_PACKET:
		udcb->ctrl_state = EP0_STATE_TEST_PACKET;
		break;
	default:
		return -1;
	}

	return 0;
}

static void usbb_ep0_tx_complete(struct udc *udc, struct usbb_udc_ep *ep)
{
	struct usb_request	*req;
	struct buffer		*buf;
	unsigned int		buf_offset;
	unsigned int		bytes_written;

	if (slist_is_empty(&ep->req_queue)) {
		assert(slist_is_empty(&ep->buf_queue));
		return;
	}

	assert(!slist_is_empty(&ep->buf_queue));

	req = slist_peek_head(&ep->req_queue, struct usb_request, node);
	buf = slist_peek_head(&ep->buf_queue, struct buffer, node);

	buf_offset = ep->buf_offset;
	bytes_written = ep->bytes_written;

	req->bytes_xfered += bytes_written;

	while (bytes_written && buf_offset + bytes_written >= buf->len) {
		bytes_written -= buf->len - buf_offset;
		buf_offset = 0;
		slist_pop_head_node(&ep->buf_queue);
		if (bytes_written)
			assert(!slist_is_empty(&ep->buf_queue));
		buf = slist_peek_head(&ep->buf_queue, struct buffer, node);
	}

	if (slist_is_empty(&ep->buf_queue)) {
		slist_init(&ep->req_queue);
		/* The queue is now empty */
		clear_bit(USBB_EP_ACTIVE_XFER, &ep->flags);
		slist_give_back_head(&req->buf_list, &ep->buf_queue);
		usbb_udc_req_done(udc, req, 0);
	}

	ep->buf_offset = bytes_written;
	ep->bytes_written = 0;
}

static void usbb_ep0_tx_flush(struct udc *udc, struct usbb_udc_ep *ep)
{
	/* First, complete all buffers that have been submitted */
	usbb_ep0_tx_complete(udc, ep);

	/* Then, if the request is still not done, terminate it */
	if (!slist_is_empty(&ep->req_queue)) {
		struct usb_request	*req;

		req = slist_pop_head(&ep->req_queue, struct usb_request, node);
		assert(slist_is_empty(&ep->req_queue));
		slist_init(&ep->buf_queue);
		ep->buf_offset = 0;
		clear_bit(USBB_EP_ACTIVE_XFER, &ep->flags);
		usbb_udc_req_done(udc, req, -STATUS_PROTO_ERROR);
	}
}

static void usbb_ep0_interrupt(struct usbb_udc *udcb)
{
	struct usbb_udc_ep	*ep;
	uint32_t		status;
	uint32_t		control;
	uint32_t		pending;

	status = usbb_udc_read_reg(UESTA(0));
	control = usbb_udc_read_reg(UECON(0));
	pending = status & control;

	ep = &udcb->ep[0];

	if (pending & USBB_EP_RXOUTI) {
		struct usb_request	*req;
		struct buffer		*buf;
		unsigned int		buf_offset;
		unsigned int		fifo_offset = 0;
		unsigned int		fifo_len;

		if (slist_is_empty(&ep->req_queue)) {
			assert(slist_is_empty(&ep->buf_queue));

			/* Unexpected data -- stall the endpoint */
			goto stall;
		}

		req = slist_peek_head(&ep->req_queue, struct usb_request, node);

		buf_offset = ep->buf_offset;
		fifo_len = USBB_UESTA_GET_BYCT(status);
		while (fifo_offset < fifo_len
				&& !slist_is_empty(&ep->buf_queue)) {
			unsigned int nbytes;

			buf = slist_peek_head(&ep->buf_queue,
					struct buffer, node);

			nbytes = min(fifo_len - fifo_offset,
					buf->len - buf_offset);
			memcpy(buf->addr.ptr + buf_offset,
					usbb_ep_fifo(0) + fifo_offset,
					nbytes);

			buf_offset += nbytes;
			fifo_offset += nbytes;

			if (buf_offset == buf->len) {
				req->bytes_xfered += buf_offset;
				slist_pop_head_node(&ep->buf_queue);

				buf_offset = 0;
			}
		}

		usbb_udc_write_reg(UESTACLR(0), USBB_EP_RXOUTI);
		ep->buf_offset = buf_offset;

		if (slist_is_empty(&ep->buf_queue)) {
			/* Request is done */
			assert(buf_offset == 0);
			slist_init(&ep->req_queue);
			usbb_udc_write_reg(UECONCLR(0), USBB_EP_RXOUTI);
			udcb->ctrl_state = EP0_STATE_SETUP;
			clear_bit(USBB_EP_ACTIVE_XFER, &ep->flags);
			usbb_udc_req_done(&udcb->udc, req, 0);
		}
	}
	if (pending & USBB_EP_TXINI) {
		struct usb_request	*req;
		struct buffer		*buf;
		unsigned int		buf_offset;
		void			*fifo;
		void			*fifo_end;

		if (usbb_udc_entering_test_mode(udcb)) {
			/* Status IN packet after SetFeature(TEST_X) sent */
			usbb_udc_do_test_mode(udcb);
			/* Don't re-enable ep0 interrupt here */
			return;
		}

		if (udcb->ctrl_state == EP0_STATE_STATUS_IN) {
			struct usb_setup_req	*setup;

			/* Status IN complete */
			setup = &udcb->setup_req;
			if (setup->bRequest == USB_REQ_SET_ADDRESS) {
				uint32_t udcon;
				uint16_t addr = le16_to_cpu(setup->wValue);

				udcon = usbb_udc_read_reg(UDCON);
				udcon &= ~USBB_UDCON_UADD_MASK;
				udcon |= USBB_UDCON_UADD(addr);
				udcon |= USBB_UDCON_ADDEN;
				usbb_udc_write_reg(UDCON, udcon);
				udcb->udc.address = addr;
			}
			usbb_udc_write_reg(UECONCLR(0), USBB_EP_TXINI);
			udcb->ctrl_state = EP0_STATE_SETUP;
			goto out;
		}

		/* Send ZLP to terminate the current transfer if required */
		if (udcb->ctrl_state == EP0_STATE_DATA_ZLP) {
			udcb->ctrl_state = EP0_STATE_DATA_IN;
			usbb_udc_write_reg(UESTACLR(0), USBB_EP_TXINI);
			goto out;
		}

		/* Terminate any completed buffers */
		usbb_ep0_tx_complete(&udcb->udc, ep);

		/* Submit any remaining buffers */
		if (slist_is_empty(&ep->buf_queue)) {
			usbb_udc_write_reg(UECONCLR(0), USBB_EP_TXINI);
			udcb->ctrl_state = EP0_STATE_SETUP;
			goto out;
		}

		buf_offset = ep->buf_offset;
		buf = slist_peek_head(&ep->buf_queue, struct buffer, node);

		fifo = usbb_ep_fifo(0);
		fifo_end = fifo + CONFIG_USB_FUNC_MAXPACKETSIZE0;
		while (fifo < fifo_end) {
			unsigned int nbytes;
			unsigned int buf_len;

			buf_len = buf->len;

			nbytes = min(fifo_end - fifo, buf_len - buf_offset);
			dbg_printf("fifo %p data %p off %u nbytes %u\n",
					fifo, buf->addr.ptr,
					buf_offset, nbytes);
			memcpy(fifo, buf->addr.ptr + buf_offset, nbytes);

			buf_offset += nbytes;
			fifo += nbytes;

			if (buf_offset == buf_len) {
				if (slist_node_is_last(&ep->buf_queue,
							&buf->node))
					break;
				buf = slist_peek_next(&buf->node,
						struct buffer, node);
				buf_offset = 0;
			}
		}

		ep->bytes_written = CONFIG_USB_FUNC_MAXPACKETSIZE0
			- ((unsigned long)fifo_end - (unsigned long)fifo);

		req = slist_peek_head(&ep->req_queue, struct usb_request, node);
		if (&buf->node == req->buf_list.last
				&& buf_offset == buf->len
				&& test_bit(USB_REQ_SHORT_PKT, &req->flags)
				&& fifo == fifo_end)
			udcb->ctrl_state = EP0_STATE_DATA_ZLP;

		usbb_udc_write_reg(UESTACLR(0), USBB_EP_TXINI);
	}
	if (pending & USBB_EP_RXSTPI) {
		struct usb_setup_req	*setup;

		setup = &udcb->setup_req;

		/*
		 * A SETUP packet clears the TX complete interrupt, so
		 * complete all submitted requests now.
		 */
		if (control & USBB_EP_TXINI) {
			usbb_udc_write_reg(UECONCLR(0), USBB_EP_TXINI);
			usbb_ep0_tx_flush(&udcb->udc, ep);
		}

		if (USBB_UESTA_GET_BYCT(status) != sizeof(*setup)) {
			/* Invalid SETUP packet length -- stall */
			usbb_udc_write_reg(UESTACLR(0), USBB_EP_RXSTPI);
			goto stall;
		}

		memcpy(setup, usbb_ep_fifo(0), sizeof(*setup));
		usbb_udc_write_reg(UESTACLR(0), USBB_EP_RXSTPI);

		if (udc_lib_process_setup_request(&udcb->udc, setup) < 0)
			goto stall;
	}

out:
	usbb_udc_write_reg(UDINTESET, USBB_UDINT_EP(0));
	return;

stall:
	usbb_udc_write_reg(UECONCLR(0), USBB_EP_TXINI | USBB_EP_RXOUTI);
	udc_ep_set_halt(&udcb->udc, 0);
	goto out;
}

/**
 * \internal
 * \brief Handle a DMA interrupt for an endpoint.
 */
static void usbb_udc_dma_interrupt(struct usbb_udc *udcb, usb_ep_id_t ep_id)
{
	struct usbb_udc_ep	*ep = &udcb->ep[ep_id];
	struct buffer		*buf;
	struct usb_request	*req;
	phys_addr_t		next;
	uint32_t		status;
	uint32_t		ctrl;

	next = usbb_udc_dma_read_reg(ep_id, NEXTDESC);
	status = usbb_udc_dma_read_reg(ep_id, STATUS);
	ctrl = usbb_udc_dma_read_reg(ep_id, CONTROL);

	dbg_printf("dma%u n:%08lx s:%08x c:%08x\n", ep_id, next, status, ctrl);

	if (status & USBB_DMA_CH_ACTIVE) {
		assert(!slist_is_empty(&ep->buf_queue));

		req = NULL;
		do {
			struct usbb_sw_dma_desc	*dma_desc;

			if (!req) {
				assert(!slist_is_empty(&ep->req_queue));
				req = slist_pop_head(&ep->req_queue,
						struct usb_request, node);
			}

			buf = slist_pop_head(&ep->buf_queue,
					struct buffer, node);
			dma_desc = buf->dma_desc;

			/* DMA still active, so must have a descriptor */
			assert(dma_desc);

			if (slist_is_empty(&ep->buf_queue)
					|| dma_desc->hw.next == next) {
				slist_insert_head(&ep->buf_queue, &buf->node);
				slist_insert_head(&ep->req_queue, &req->node);
				break;
			}

			req->bytes_xfered += buf->len;
			usbb_udc_dma_buf_done(buf);

			if (&buf->node == req->buf_list.last) {
				usbb_udc_dma_req_done(&udcb->udc,
						&ep->buf_queue, req, 0);
				req = NULL;
			}
		} while (!slist_is_empty(&ep->buf_queue));

		dbg_verbose("still active\n");
		assert(!slist_is_empty(&ep->buf_queue));
		usbb_udc_write_reg(UDINTESET, USBB_UDINT_DMA(ep_id));
	} else if (!slist_is_empty(&ep->buf_queue)) {
		req = NULL;
		do {
			if (!req) {
				assert(!slist_is_empty(&ep->req_queue));
				req = slist_pop_head(&ep->req_queue,
						struct usb_request, node);
			}

			buf = slist_pop_head(&ep->buf_queue,
					struct buffer, node);
			if (unlikely(!buf->dma_desc)) {
				slist_insert_head(&ep->buf_queue, &buf->node);
				slist_insert_head(&ep->req_queue, &req->node);
				break;
			}

			req->bytes_xfered += buf->len;

			usbb_udc_dma_buf_done(buf);

			if (&buf->node == req->buf_list.last) {
				struct buffer	*next_buf;

				next_buf = slist_peek_head(&ep->buf_queue,
						struct buffer, node);
				if ((slist_is_empty(&ep->buf_queue)
							|| !next_buf->dma_desc)
						&& (status & USBB_DMA_EOT)) {
					size_t len;
					len = USBB_DMA_GET_BYTE_LEN(status);
					req->bytes_xfered -= len;
				}

				usbb_udc_dma_req_done(&udcb->udc,
						&ep->buf_queue, req, 0);
				req = NULL;
			}
		} while (!slist_is_empty(&ep->buf_queue));

		if (slist_is_empty(&ep->req_queue))
			assert(slist_is_empty(&ep->buf_queue));
		else
			assert(!slist_is_empty(&ep->buf_queue));

		clear_bit(USBB_EP_ACTIVE_XFER, &ep->flags);

		if (usb_ep_is_in(ep->desc))
			usbb_udc_submit_in_queue(udcb, ep_id, ep);
		else
			usbb_udc_submit_out_queue(udcb, ep_id, ep);
	}

}

/**
 * \internal
 * \brief The USBB device-mode soft interrupt handler.
 */
static void usbb_udc_softirq(void *data)
{
	struct usbb_udc	*udcb = data;
	struct udc	*udc = &udcb->udc;
	uint32_t	udint;
	unsigned int	i;

	udint = usbb_udc_read_reg(UDINT);

	if (udint & USBB_UDINT_EORST) {
		uint32_t	usbsta;

		usbb_udc_write_reg(UDINTCLR, USBB_UDINT_EORST);

		/* Reset the device state */
		udc->address = 0;
		clear_bit(USB_DEV_IS_SUSPENDED, &udc->flags);

		/* Figure out what speed we're running at */
		usbsta = usbb_read_reg(USBSTA);
		switch (usbsta & USBB_USBSTA_SPEED_MASK) {
		case USBB_USBSTA_SPEED_LOW:
			udc->speed = USB_SPEED_LOW;
			break;
		case USBB_USBSTA_SPEED_FULL:
			udc->speed = USB_SPEED_FULL;
			break;
		case USBB_USBSTA_SPEED_HIGH:
			udc->speed = USB_SPEED_HIGH;
			break;
		default:
			udc->speed = USB_SPEED_UNKNOWN;
			goto out;
		}

		dbg_printf("usbb-udc: reset speed %u usbsta %08x\n",
				udc->speed, usbsta);

		usbb_ep0_tx_flush(udc, &udcb->ep[0]);
		usb_func_reset(udc);

		/* Set up ep0 for control transfers */
		if (usbb_udc_configure_ep(0, CONFIG_USB_FUNC_MAXPACKETSIZE0,
					USB_EP_XFER_CONTROL, 0, 1, false)) {
			udc->speed = USB_SPEED_UNKNOWN;
			goto out;
		}

		usbb_udc_write_reg(UDINTESET, USBB_UDINT_EP(0));

		/* Get ready to be enumerated */
		udcb->ctrl_state = EP0_STATE_SETUP;
		usbb_udc_write_reg(UECONSET(0), USBB_EP_RXSTPI);
	} else {
		if (udint & USBB_UDINT_EP(0))
			usbb_ep0_interrupt(udcb);

		for (i = 1; i < CONFIG_USBB_NR_EP; i++) {
			udint = usbb_udc_read_reg(UDINT);
			if (udint & USBB_UDINT_DMA(i))
				usbb_udc_dma_interrupt(udcb, i);
		}
	}

out:
	usbb_udc_write_reg(UDINTESET, USBB_UDINT_EORST);
}

/**
 * \internal
 * \brief The USBB device-mode interrupt handler.
 *
 * \param udcb The USBB Device Controller instance.
 */
void usbb_udc_interrupt(struct usbb_udc *udcb)
{
	usbb_udc_write_reg(UDINTECLR, usbb_udc_read_reg(UDINT));
	softirq_raise(SOFTIRQ_ID_USBB_UDC);
	usbb_udc_read_reg(UDINTE);
}

static void usbb_udc_maybe_attach(struct usbb_udc *udcb)
{
	struct udc	*udc = &udcb->udc;

	dbg_printf("usbb_udc maybe attach: flags=0x%lx\n", udc->flags);
	if (usbb_udc_is_enabled(udcb)
			&& test_bit(USB_DEV_HAS_POWER, &udc->flags)
			&& test_bit(USB_DEV_AUTOATTACH, &udc->flags)) {
		dbg_printf("usbb_udc: attaching...\n");
		usbb_udc_write_reg(UDCON,
				usbb_udc_read_reg(UDCON) & ~USBB_UDCON_DETACH);
		usbb_udc_write_reg(UDINTESET, USBB_UDINT_EORST);
	}
}

static void usbb_udc_detach(struct usbb_udc *udcb)
{
	struct udc	*udc = &udcb->udc;
	uint32_t	udcon;

	dbg_printf("usbb_udc detach: flags=0x%lx\n", udc->flags);

	udc->speed = USB_SPEED_UNKNOWN;
	udc->address = 0;
	udc->flags &= (1 << USB_DEV_IS_ENABLED) | (1 << USB_DEV_HAS_POWER)
			| (1 << USB_DEV_AUTOATTACH);

	udcon = usbb_udc_read_reg(UDCON);

	if (!(udcon & USBB_UDCON_DETACH)) {
		usb_func_reset(&udcb->udc);

		usbb_udc_write_reg(UDCON, udcon | USBB_UDCON_DETACH);
		usbb_udc_write_reg(UDINTECLR, ~0UL);
	}
}

/**
 * \internal
 * \brief Signal that a high Vbus level has been detected.
 *
 * This function is called by the USBB bus interface driver when Vbus
 * power is provided by the host.
 *
 * \param udcb The USBB Device Controller instance
 */
void usbb_udc_vbus_on(struct usbb_udc *udcb)
{
	struct udc	*udc = &udcb->udc;

	if (!test_bit(USB_DEV_HAS_POWER, &udc->flags)) {
		dbg_printf("usbb_udc: Vbus ON\n");
		set_bit(USB_DEV_HAS_POWER, &udc->flags);
		usbb_udc_maybe_attach(udcb);
	}
}

/**
 * \internal
 * \brief Signal that a low Vbus level has been detected.
 *
 * This function is called by the USBB bus interface driver when Vbus
 * power is no longer provided by the host.
 *
 * \param udcb The USBB Device Controller instance
 */
void usbb_udc_vbus_off(struct usbb_udc *udcb)
{
	struct udc	*udc = &udcb->udc;

	if (test_bit(USB_DEV_HAS_POWER, &udc->flags)) {
		dbg_printf("usbb_udc: Vbus OFF\n");

		clear_bit(USB_DEV_HAS_POWER, &udc->flags);

		usbb_udc_detach(udcb);
	}
}

/**
 * \brief Attach \a udc to the bus when possible
 *
 * Call this function to signal that the application is ready for the
 * UDC to attach to the bus. This will cause the UDC to attach whenever
 * the following conditions are present:
 * - The ID pin indicates Device operation. When the driver operates in
 *   device-only mode, this condition is assumed to always be true.
 * - An acceptable Vbus level from the host is detected.
 */
void udc_attach(struct udc *udc)
{
	struct usbb_udc	*udcb = usbb_udc_of(udc);
	unsigned long	iflags;

	iflags = cpu_irq_save();
	if (!test_bit(USB_DEV_AUTOATTACH, &udc->flags)) {
		set_bit(USB_DEV_AUTOATTACH, &udc->flags);
		usbb_udc_maybe_attach(udcb);
	}
	cpu_irq_restore(iflags);
}

/**
 * \brief Detach \a udc from the bus
 *
 * Call this function to forcibly detach the UDC from the bus. The UDC
 * will detach immediately and won't reattach until udc_attach() is
 * called, subject to the conditions listed for that function.
 */
void udc_detach(struct udc *udc)
{
	struct usbb_udc	*udcb = usbb_udc_of(udc);
	unsigned long	iflags;

	iflags = cpu_irq_save();
	if (test_bit(USB_DEV_AUTOATTACH, &udc->flags)) {
		clear_bit(USB_DEV_AUTOATTACH, &udc->flags);
		usbb_udc_detach(udcb);
	}
	cpu_irq_restore(iflags);
}

static struct usbb_udc the_usbb_udc;

/**
 * \internal
 * \brief Initialize the device part of the USBB controller.
 *
 * This function does any device-side initialization necessary when
 * the USBB controller as a whole is being initialized. It does not
 * enable any device-side functionality.
 *
 * \return A USBB device controller instance.
 */
struct usbb_udc *usbb_udc_init(void)
{
	struct usbb_udc		*udcb = &the_usbb_udc;

	slist_init(&udcb->ep[0].req_queue);
	slist_init(&udcb->ep[0].buf_queue);

	udcb->udc.flags = 0;

#if defined(CHIP_USBB_UDC_HIGH_SPEED) && !defined(CONFIG_UDC_HIGH_SPEED)
	/* Force full-speed mode */
	usbb_udc_write_reg(UDCON, USBB_UDCON_SPDCONF_FULL);
#elif defined(CONFIG_UDC_LOW_SPEED)
	usbb_udc_write_reg(UDCON, USBB_UDCON_LS);
#else
	usbb_udc_write_reg(UDCON, 0);
#endif

	softirq_set_handler(SOFTIRQ_ID_USBB_UDC, usbb_udc_softirq, udcb);

	return udcb;
}

/**
 * \internal
 * \brief Shut down the device part of the USBB controller.
 *
 * This function does any device-side cleanups necessary when the USBB
 * controller as a whole is being shut down.
 *
 * \param udcb The USBB device controller instance.
 */
void usbb_udc_shutdown(struct usbb_udc *udcb)
{
	udcb->udc.flags = 0;
}
