ioat_dma.c

linux 内核源代码

/*
 * Intel I/OAT DMA Linux driver
 * Copyright(c) 2004 - 2007 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * The full GNU General Public License is included in this distribution in
 * the file called "COPYING".
 *
 */

/*
 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
 * copy operations.
 */

#include <linux/init.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/dmaengine.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include "ioatdma.h"
#include "ioatdma_registers.h"
#include "ioatdma_hw.h"

#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)

static int ioat_pending_level = 4;
module_param(ioat_pending_level, int, 0644);
MODULE_PARM_DESC(ioat_pending_level,
		 "high-water mark for pushing ioat descriptors (default: 4)");

/* internal functions */
static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan);
static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);

static struct ioat_desc_sw *
ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
static struct ioat_desc_sw *
ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);

static inline struct ioat_dma_chan *ioat_lookup_chan_by_index(
						struct ioatdma_device *device,
						int index)
{
	return device->idx[index];
}

/**
 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
 * @irq: interrupt id
 * @data: interrupt data
 */
static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
{
	struct ioatdma_device *instance = data;
	struct ioat_dma_chan *ioat_chan;
	unsigned long attnstatus;
	int bit;
	u8 intrctrl;

	intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);

	if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
		return IRQ_NONE;

	if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
		writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
		return IRQ_NONE;
	}

	attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
	for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
		ioat_chan = ioat_lookup_chan_by_index(instance, bit);
		tasklet_schedule(&ioat_chan->cleanup_task);
	}

	writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
	return IRQ_HANDLED;
}

/**
 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
 * @irq: interrupt id
 * @data: interrupt data
 */
static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
{
	struct ioat_dma_chan *ioat_chan = data;

	tasklet_schedule(&ioat_chan->cleanup_task);

	return IRQ_HANDLED;
}

static void ioat_dma_cleanup_tasklet(unsigned long data);

/**
 * ioat_dma_enumerate_channels - find and initialize the device's channels
 * @device: the device to be enumerated
 */
static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
{
	u8 xfercap_scale;
	u32 xfercap;
	int i;
	struct ioat_dma_chan *ioat_chan;

	device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
	xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
	xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));

	for (i = 0; i < device->common.chancnt; i++) {
		ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
		if (!ioat_chan) {
			device->common.chancnt = i;
			break;
		}

		ioat_chan->device = device;
		ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
		ioat_chan->xfercap = xfercap;
		ioat_chan->desccount = 0;
		if (ioat_chan->device->version != IOAT_VER_1_2) {
			writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE
					| IOAT_DMA_DCA_ANY_CPU,
				ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
		}
		spin_lock_init(&ioat_chan->cleanup_lock);
		spin_lock_init(&ioat_chan->desc_lock);
		INIT_LIST_HEAD(&ioat_chan->free_desc);
		INIT_LIST_HEAD(&ioat_chan->used_desc);
		/* This should be made common somewhere in dmaengine.c */
		ioat_chan->common.device = &device->common;
		list_add_tail(&ioat_chan->common.device_node,
			      &device->common.channels);
		device->idx[i] = ioat_chan;
		tasklet_init(&ioat_chan->cleanup_task,
			     ioat_dma_cleanup_tasklet,
			     (unsigned long) ioat_chan);
		tasklet_disable(&ioat_chan->cleanup_task);
	}
	return device->common.chancnt;
}

static void ioat_set_src(dma_addr_t addr,
			 struct dma_async_tx_descriptor *tx,
			 int index)
{
	tx_to_ioat_desc(tx)->src = addr;
}

static void ioat_set_dest(dma_addr_t addr,
			  struct dma_async_tx_descriptor *tx,
			  int index)
{
	tx_to_ioat_desc(tx)->dst = addr;
}

/**
 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
 *                                 descriptors to hw
 * @chan: DMA channel handle
 */
static inline void __ioat1_dma_memcpy_issue_pending(
						struct ioat_dma_chan *ioat_chan)
{
	ioat_chan->pending = 0;
	writeb(IOAT_CHANCMD_APPEND, ioat_chan->reg_base + IOAT1_CHANCMD_OFFSET);
}

static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
{
	struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);

	if (ioat_chan->pending != 0) {
		spin_lock_bh(&ioat_chan->desc_lock);
		__ioat1_dma_memcpy_issue_pending(ioat_chan);
		spin_unlock_bh(&ioat_chan->desc_lock);
	}
}

static inline void __ioat2_dma_memcpy_issue_pending(
						struct ioat_dma_chan *ioat_chan)
{
	ioat_chan->pending = 0;
	writew(ioat_chan->dmacount,
	       ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
}

static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)
{
	struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);

	if (ioat_chan->pending != 0) {
		spin_lock_bh(&ioat_chan->desc_lock);
		__ioat2_dma_memcpy_issue_pending(ioat_chan);
		spin_unlock_bh(&ioat_chan->desc_lock);
	}
}

static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
{
	struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
	struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
	struct ioat_desc_sw *prev, *new;
	struct ioat_dma_descriptor *hw;
	dma_cookie_t cookie;
	LIST_HEAD(new_chain);
	u32 copy;
	size_t len;
	dma_addr_t src, dst;
	int orig_ack;
	unsigned int desc_count = 0;

	/* src and dest and len are stored in the initial descriptor */
	len = first->len;
	src = first->src;
	dst = first->dst;
	orig_ack = first->async_tx.ack;
	new = first;

	spin_lock_bh(&ioat_chan->desc_lock);
	prev = to_ioat_desc(ioat_chan->used_desc.prev);
	prefetch(prev->hw);
	do {
		copy = min_t(size_t, len, ioat_chan->xfercap);

		new->async_tx.ack = 1;

		hw = new->hw;
		hw->size = copy;
		hw->ctl = 0;
		hw->src_addr = src;
		hw->dst_addr = dst;
		hw->next = 0;

		/* chain together the physical address list for the HW */
		wmb();
		prev->hw->next = (u64) new->async_tx.phys;

		len -= copy;
		dst += copy;
		src += copy;

		list_add_tail(&new->node, &new_chain);
		desc_count++;
		prev = new;
	} while (len && (new = ioat1_dma_get_next_descriptor(ioat_chan)));

	hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
	if (new->async_tx.callback) {
		hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
		if (first != new) {
			/* move callback into to last desc */
			new->async_tx.callback = first->async_tx.callback;
			new->async_tx.callback_param
					= first->async_tx.callback_param;
			first->async_tx.callback = NULL;
			first->async_tx.callback_param = NULL;
		}
	}

	new->tx_cnt = desc_count;
	new->async_tx.ack = orig_ack; /* client is in control of this ack */

	/* store the original values for use in later cleanup */
	if (new != first) {
		new->src = first->src;
		new->dst = first->dst;
		new->len = first->len;
	}

	/* cookie incr and addition to used_list must be atomic */
	cookie = ioat_chan->common.cookie;
	cookie++;
	if (cookie < 0)
		cookie = 1;
	ioat_chan->common.cookie = new->async_tx.cookie = cookie;

	/* write address into NextDescriptor field of last desc in chain */
	to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
							first->async_tx.phys;
	__list_splice(&new_chain, ioat_chan->used_desc.prev);

	ioat_chan->dmacount += desc_count;
	ioat_chan->pending += desc_count;
	if (ioat_chan->pending >= ioat_pending_level)
		__ioat1_dma_memcpy_issue_pending(ioat_chan);
	spin_unlock_bh(&ioat_chan->desc_lock);

	return cookie;
}

static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)
{
	struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
	struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
	struct ioat_desc_sw *new;
	struct ioat_dma_descriptor *hw;
	dma_cookie_t cookie;
	u32 copy;
	size_t len;
	dma_addr_t src, dst;
	int orig_ack;
	unsigned int desc_count = 0;

	/* src and dest and len are stored in the initial descriptor */
	len = first->len;
	src = first->src;
	dst = first->dst;
	orig_ack = first->async_tx.ack;
	new = first;

	/*
	 * ioat_chan->desc_lock is still in force in version 2 path
	 * it gets unlocked at end of this function
	 */
	do {
		copy = min_t(size_t, len, ioat_chan->xfercap);

		new->async_tx.ack = 1;

		hw = new->hw;
		hw->size = copy;
		hw->ctl = 0;
		hw->src_addr = src;
		hw->dst_addr = dst;

		len -= copy;
		dst += copy;
		src += copy;
		desc_count++;
	} while (len && (new = ioat2_dma_get_next_descriptor(ioat_chan)));

	hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
	if (new->async_tx.callback) {
		hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
		if (first != new) {
			/* move callback into to last desc */
			new->async_tx.callback = first->async_tx.callback;
			new->async_tx.callback_param
					= first->async_tx.callback_param;
			first->async_tx.callback = NULL;
			first->async_tx.callback_param = NULL;
		}
	}

	new->tx_cnt = desc_count;
	new->async_tx.ack = orig_ack; /* client is in control of this ack */

	/* store the original values for use in later cleanup */
	if (new != first) {
		new->src = first->src;
		new->dst = first->dst;
		new->len = first->len;
	}

	/* cookie incr and addition to used_list must be atomic */
	cookie = ioat_chan->common.cookie;
	cookie++;
	if (cookie < 0)
		cookie = 1;
	ioat_chan->common.cookie = new->async_tx.cookie = cookie;

	ioat_chan->dmacount += desc_count;
	ioat_chan->pending += desc_count;
	if (ioat_chan->pending >= ioat_pending_level)
		__ioat2_dma_memcpy_issue_pending(ioat_chan);
	spin_unlock_bh(&ioat_chan->desc_lock);

	return cookie;
}

/**
 * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
 * @ioat_chan: the channel supplying the memory pool for the descriptors
 * @flags: allocation flags
 */
static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
					struct ioat_dma_chan *ioat_chan,
					gfp_t flags)
{
	struct ioat_dma_descriptor *desc;
	struct ioat_desc_sw *desc_sw;
	struct ioatdma_device *ioatdma_device;
	dma_addr_t phys;

	ioatdma_device = to_ioatdma_device(ioat_chan->common.device);
	desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
	if (unlikely(!desc))
		return NULL;

	desc_sw = kzalloc(sizeof(*desc_sw), flags);
	if (unlikely(!desc_sw)) {
		pci_pool_free(ioatdma_device->dma_pool, desc, phys);
		return NULL;
	}

	memset(desc, 0, sizeof(*desc));
	dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
	desc_sw->async_tx.tx_set_src = ioat_set_src;
	desc_sw->async_tx.tx_set_dest = ioat_set_dest;
	switch (ioat_chan->device->version) {
	case IOAT_VER_1_2:
		desc_sw->async_tx.tx_submit = ioat1_tx_submit;
		break;
	case IOAT_VER_2_0:
		desc_sw->async_tx.tx_submit = ioat2_tx_submit;
		break;
	}
	INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);

	desc_sw->hw = desc;
	desc_sw->async_tx.phys = phys;

	return desc_sw;
}

static int ioat_initial_desc_count = 256;
module_param(ioat_initial_desc_count, int, 0644);
MODULE_PARM_DESC(ioat_initial_desc_count,
		 "initial descriptors per channel (default: 256)");

/**
 * ioat2_dma_massage_chan_desc - link the descriptors into a circle
 * @ioat_chan: the channel to be massaged
 */
static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat_chan)
{
	struct ioat_desc_sw *desc, *_desc;

	/* setup used_desc */
	ioat_chan->used_desc.next = ioat_chan->free_desc.next;
	ioat_chan->used_desc.prev = NULL;

	/* pull free_desc out of the circle so that every node is a hw
	 * descriptor, but leave it pointing to the list
	 */
	ioat_chan->free_desc.prev->next = ioat_chan->free_desc.next;
	ioat_chan->free_desc.next->prev = ioat_chan->free_desc.prev;

	/* circle link the hw descriptors */
	desc = to_ioat_desc(ioat_chan->free_desc.next);
	desc->hw->next = to_ioat_desc(desc->node.next)->async_tx.phys;
	list_for_each_entry_safe(desc, _desc, ioat_chan->free_desc.next, node) {
		desc->hw->next = to_ioat_desc(desc->node.next)->async_tx.phys;
	}
}