pci-calgary_64.c

linux 内核源代码

/*
 * Derived from arch/powerpc/kernel/iommu.c
 *
 * Copyright IBM Corporation, 2006-2007
 * Copyright (C) 2006  Jon Mason <[e-mail]>
 *
 * Author: Jon Mason <[e-mail]>
 * Author: Muli Ben-Yehuda <[e-mail]>

 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/dma-mapping.h>
#include <linux/init.h>
#include <linux/bitops.h>
#include <linux/pci_ids.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/scatterlist.h>
#include <asm/gart.h>
#include <asm/calgary.h>
#include <asm/tce.h>
#include <asm/pci-direct.h>
#include <asm/system.h>
#include <asm/dma.h>
#include <asm/rio.h>

#ifdef CONFIG_CALGARY_IOMMU_ENABLED_BY_DEFAULT
int use_calgary __read_mostly = 1;
#else
int use_calgary __read_mostly = 0;
#endif /* CONFIG_CALGARY_DEFAULT_ENABLED */

#define PCI_DEVICE_ID_IBM_CALGARY 0x02a1
#define PCI_DEVICE_ID_IBM_CALIOC2 0x0308

/* register offsets inside the host bridge space */
#define CALGARY_CONFIG_REG	0x0108
#define PHB_CSR_OFFSET		0x0110 /* Channel Status */
#define PHB_PLSSR_OFFSET	0x0120
#define PHB_CONFIG_RW_OFFSET	0x0160
#define PHB_IOBASE_BAR_LOW	0x0170
#define PHB_IOBASE_BAR_HIGH	0x0180
#define PHB_MEM_1_LOW		0x0190
#define PHB_MEM_1_HIGH		0x01A0
#define PHB_IO_ADDR_SIZE	0x01B0
#define PHB_MEM_1_SIZE		0x01C0
#define PHB_MEM_ST_OFFSET	0x01D0
#define PHB_AER_OFFSET		0x0200
#define PHB_CONFIG_0_HIGH	0x0220
#define PHB_CONFIG_0_LOW	0x0230
#define PHB_CONFIG_0_END	0x0240
#define PHB_MEM_2_LOW		0x02B0
#define PHB_MEM_2_HIGH		0x02C0
#define PHB_MEM_2_SIZE_HIGH	0x02D0
#define PHB_MEM_2_SIZE_LOW	0x02E0
#define PHB_DOSHOLE_OFFSET	0x08E0

/* CalIOC2 specific */
#define PHB_SAVIOR_L2		0x0DB0
#define PHB_PAGE_MIG_CTRL	0x0DA8
#define PHB_PAGE_MIG_DEBUG	0x0DA0
#define PHB_ROOT_COMPLEX_STATUS 0x0CB0

/* PHB_CONFIG_RW */
#define PHB_TCE_ENABLE		0x20000000
#define PHB_SLOT_DISABLE	0x1C000000
#define PHB_DAC_DISABLE		0x01000000
#define PHB_MEM2_ENABLE		0x00400000
#define PHB_MCSR_ENABLE		0x00100000
/* TAR (Table Address Register) */
#define TAR_SW_BITS		0x0000ffffffff800fUL
#define TAR_VALID		0x0000000000000008UL
/* CSR (Channel/DMA Status Register) */
#define CSR_AGENT_MASK		0xffe0ffff
/* CCR (Calgary Configuration Register) */
#define CCR_2SEC_TIMEOUT	0x000000000000000EUL
/* PMCR/PMDR (Page Migration Control/Debug Registers */
#define PMR_SOFTSTOP		0x80000000
#define PMR_SOFTSTOPFAULT	0x40000000
#define PMR_HARDSTOP		0x20000000

#define MAX_NUM_OF_PHBS		8 /* how many PHBs in total? */
#define MAX_NUM_CHASSIS		8 /* max number of chassis */
/* MAX_PHB_BUS_NUM is the maximal possible dev->bus->number */
#define MAX_PHB_BUS_NUM		(MAX_NUM_OF_PHBS * MAX_NUM_CHASSIS * 2)
#define PHBS_PER_CALGARY	4

/* register offsets in Calgary's internal register space */
static const unsigned long tar_offsets[] = {
	0x0580 /* TAR0 */,
	0x0588 /* TAR1 */,
	0x0590 /* TAR2 */,
	0x0598 /* TAR3 */
};

static const unsigned long split_queue_offsets[] = {
	0x4870 /* SPLIT QUEUE 0 */,
	0x5870 /* SPLIT QUEUE 1 */,
	0x6870 /* SPLIT QUEUE 2 */,
	0x7870 /* SPLIT QUEUE 3 */
};

static const unsigned long phb_offsets[] = {
	0x8000 /* PHB0 */,
	0x9000 /* PHB1 */,
	0xA000 /* PHB2 */,
	0xB000 /* PHB3 */
};

/* PHB debug registers */

static const unsigned long phb_debug_offsets[] = {
	0x4000	/* PHB 0 DEBUG */,
	0x5000	/* PHB 1 DEBUG */,
	0x6000	/* PHB 2 DEBUG */,
	0x7000	/* PHB 3 DEBUG */
};

/*
 * STUFF register for each debug PHB,
 * byte 1 = start bus number, byte 2 = end bus number
 */

#define PHB_DEBUG_STUFF_OFFSET	0x0020

#define EMERGENCY_PAGES 32 /* = 128KB */

unsigned int specified_table_size = TCE_TABLE_SIZE_UNSPECIFIED;
static int translate_empty_slots __read_mostly = 0;
static int calgary_detected __read_mostly = 0;

static struct rio_table_hdr	*rio_table_hdr __initdata;
static struct scal_detail	*scal_devs[MAX_NUMNODES] __initdata;
static struct rio_detail	*rio_devs[MAX_NUMNODES * 4] __initdata;

struct calgary_bus_info {
	void *tce_space;
	unsigned char translation_disabled;
	signed char phbid;
	void __iomem *bbar;
};

static void calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
static void calgary_tce_cache_blast(struct iommu_table *tbl);
static void calgary_dump_error_regs(struct iommu_table *tbl);
static void calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
static void calioc2_tce_cache_blast(struct iommu_table *tbl);
static void calioc2_dump_error_regs(struct iommu_table *tbl);

static struct cal_chipset_ops calgary_chip_ops = {
	.handle_quirks = calgary_handle_quirks,
	.tce_cache_blast = calgary_tce_cache_blast,
	.dump_error_regs = calgary_dump_error_regs
};

static struct cal_chipset_ops calioc2_chip_ops = {
	.handle_quirks = calioc2_handle_quirks,
	.tce_cache_blast = calioc2_tce_cache_blast,
	.dump_error_regs = calioc2_dump_error_regs
};

static struct calgary_bus_info bus_info[MAX_PHB_BUS_NUM] = { { NULL, 0, 0 }, };

/* enable this to stress test the chip's TCE cache */
#ifdef CONFIG_IOMMU_DEBUG
int debugging __read_mostly = 1;

static inline unsigned long verify_bit_range(unsigned long* bitmap,
	int expected, unsigned long start, unsigned long end)
{
	unsigned long idx = start;

	BUG_ON(start >= end);

	while (idx < end) {
		if (!!test_bit(idx, bitmap) != expected)
			return idx;
		++idx;
	}

	/* all bits have the expected value */
	return ~0UL;
}
#else /* debugging is disabled */
int debugging __read_mostly = 0;

static inline unsigned long verify_bit_range(unsigned long* bitmap,
	int expected, unsigned long start, unsigned long end)
{
	return ~0UL;
}

#endif /* CONFIG_IOMMU_DEBUG */

static inline unsigned int num_dma_pages(unsigned long dma, unsigned int dmalen)
{
	unsigned int npages;

	npages = PAGE_ALIGN(dma + dmalen) - (dma & PAGE_MASK);
	npages >>= PAGE_SHIFT;

	return npages;
}

static inline int translation_enabled(struct iommu_table *tbl)
{
	/* only PHBs with translation enabled have an IOMMU table */
	return (tbl != NULL);
}

static void iommu_range_reserve(struct iommu_table *tbl,
	unsigned long start_addr, unsigned int npages)
{
	unsigned long index;
	unsigned long end;
	unsigned long badbit;
	unsigned long flags;

	index = start_addr >> PAGE_SHIFT;

	/* bail out if we're asked to reserve a region we don't cover */
	if (index >= tbl->it_size)
		return;

	end = index + npages;
	if (end > tbl->it_size) /* don't go off the table */
		end = tbl->it_size;

	spin_lock_irqsave(&tbl->it_lock, flags);

	badbit = verify_bit_range(tbl->it_map, 0, index, end);
	if (badbit != ~0UL) {
		if (printk_ratelimit())
			printk(KERN_ERR "Calgary: entry already allocated at "
			       "0x%lx tbl %p dma 0x%lx npages %u\n",
			       badbit, tbl, start_addr, npages);
	}

	set_bit_string(tbl->it_map, index, npages);

	spin_unlock_irqrestore(&tbl->it_lock, flags);
}

static unsigned long iommu_range_alloc(struct iommu_table *tbl,
	unsigned int npages)
{
	unsigned long flags;
	unsigned long offset;

	BUG_ON(npages == 0);

	spin_lock_irqsave(&tbl->it_lock, flags);

	offset = find_next_zero_string(tbl->it_map, tbl->it_hint,
				       tbl->it_size, npages);
	if (offset == ~0UL) {
		tbl->chip_ops->tce_cache_blast(tbl);
		offset = find_next_zero_string(tbl->it_map, 0,
					       tbl->it_size, npages);
		if (offset == ~0UL) {
			printk(KERN_WARNING "Calgary: IOMMU full.\n");
			spin_unlock_irqrestore(&tbl->it_lock, flags);
			if (panic_on_overflow)
				panic("Calgary: fix the allocator.\n");
			else
				return bad_dma_address;
		}
	}

	set_bit_string(tbl->it_map, offset, npages);
	tbl->it_hint = offset + npages;
	BUG_ON(tbl->it_hint > tbl->it_size);

	spin_unlock_irqrestore(&tbl->it_lock, flags);

	return offset;
}

static dma_addr_t iommu_alloc(struct iommu_table *tbl, void *vaddr,
	unsigned int npages, int direction)
{
	unsigned long entry;
	dma_addr_t ret = bad_dma_address;

	entry = iommu_range_alloc(tbl, npages);

	if (unlikely(entry == bad_dma_address))
		goto error;

	/* set the return dma address */
	ret = (entry << PAGE_SHIFT) | ((unsigned long)vaddr & ~PAGE_MASK);

	/* put the TCEs in the HW table */
	tce_build(tbl, entry, npages, (unsigned long)vaddr & PAGE_MASK,
		  direction);

	return ret;

error:
	printk(KERN_WARNING "Calgary: failed to allocate %u pages in "
	       "iommu %p\n", npages, tbl);
	return bad_dma_address;
}

static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
	unsigned int npages)
{
	unsigned long entry;
	unsigned long badbit;
	unsigned long badend;
	unsigned long flags;

	/* were we called with bad_dma_address? */
	badend = bad_dma_address + (EMERGENCY_PAGES * PAGE_SIZE);
	if (unlikely((dma_addr >= bad_dma_address) && (dma_addr < badend))) {
		printk(KERN_ERR "Calgary: driver tried unmapping bad DMA "
		       "address 0x%Lx\n", dma_addr);
		WARN_ON(1);
		return;
	}

	entry = dma_addr >> PAGE_SHIFT;

	BUG_ON(entry + npages > tbl->it_size);

	tce_free(tbl, entry, npages);

	spin_lock_irqsave(&tbl->it_lock, flags);

	badbit = verify_bit_range(tbl->it_map, 1, entry, entry + npages);
	if (badbit != ~0UL) {
		if (printk_ratelimit())
			printk(KERN_ERR "Calgary: bit is off at 0x%lx "
			       "tbl %p dma 0x%Lx entry 0x%lx npages %u\n",
			       badbit, tbl, dma_addr, entry, npages);
	}

	__clear_bit_string(tbl->it_map, entry, npages);

	spin_unlock_irqrestore(&tbl->it_lock, flags);
}

static inline struct iommu_table *find_iommu_table(struct device *dev)
{
	struct pci_dev *pdev;
	struct pci_bus *pbus;
	struct iommu_table *tbl;

	pdev = to_pci_dev(dev);

	pbus = pdev->bus;

	/* is the device behind a bridge? Look for the root bus */
	while (pbus->parent)
		pbus = pbus->parent;

	tbl = pci_iommu(pbus);

	BUG_ON(tbl && (tbl->it_busno != pbus->number));

	return tbl;
}

static void calgary_unmap_sg(struct device *dev,
	struct scatterlist *sglist, int nelems, int direction)
{
	struct iommu_table *tbl = find_iommu_table(dev);
	struct scatterlist *s;
	int i;

	if (!translation_enabled(tbl))
		return;

	for_each_sg(sglist, s, nelems, i) {
		unsigned int npages;
		dma_addr_t dma = s->dma_address;
		unsigned int dmalen = s->dma_length;

		if (dmalen == 0)
			break;

		npages = num_dma_pages(dma, dmalen);
		iommu_free(tbl, dma, npages);
	}
}

static int calgary_nontranslate_map_sg(struct device* dev,
	struct scatterlist *sg, int nelems, int direction)
{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nelems, i) {
		struct page *p = sg_page(s);

		BUG_ON(!p);
		s->dma_address = virt_to_bus(sg_virt(s));
		s->dma_length = s->length;
	}
	return nelems;
}

static int calgary_map_sg(struct device *dev, struct scatterlist *sg,
	int nelems, int direction)
{
	struct iommu_table *tbl = find_iommu_table(dev);
	struct scatterlist *s;
	unsigned long vaddr;
	unsigned int npages;
	unsigned long entry;
	int i;

	if (!translation_enabled(tbl))
		return calgary_nontranslate_map_sg(dev, sg, nelems, direction);

	for_each_sg(sg, s, nelems, i) {
		BUG_ON(!sg_page(s));

		vaddr = (unsigned long) sg_virt(s);
		npages = num_dma_pages(vaddr, s->length);

		entry = iommu_range_alloc(tbl, npages);
		if (entry == bad_dma_address) {
			/* makes sure unmap knows to stop */
			s->dma_length = 0;
			goto error;
		}

		s->dma_address = (entry << PAGE_SHIFT) | s->offset;

		/* insert into HW table */
		tce_build(tbl, entry, npages, vaddr & PAGE_MASK,
			  direction);

		s->dma_length = s->length;
	}

	return nelems;
error:
	calgary_unmap_sg(dev, sg, nelems, direction);
	for_each_sg(sg, s, nelems, i) {
		sg->dma_address = bad_dma_address;
		sg->dma_length = 0;
	}
	return 0;
}

static dma_addr_t calgary_map_single(struct device *dev, void *vaddr,
	size_t size, int direction)
{
	dma_addr_t dma_handle = bad_dma_address;
	unsigned long uaddr;
	unsigned int npages;
	struct iommu_table *tbl = find_iommu_table(dev);

	uaddr = (unsigned long)vaddr;
	npages = num_dma_pages(uaddr, size);

	if (translation_enabled(tbl))
		dma_handle = iommu_alloc(tbl, vaddr, npages, direction);
	else
		dma_handle = virt_to_bus(vaddr);

	return dma_handle;
}

static void calgary_unmap_single(struct device *dev, dma_addr_t dma_handle,
	size_t size, int direction)
{
	struct iommu_table *tbl = find_iommu_table(dev);
	unsigned int npages;

	if (!translation_enabled(tbl))
		return;

	npages = num_dma_pages(dma_handle, size);
	iommu_free(tbl, dma_handle, npages);
}

static void* calgary_alloc_coherent(struct device *dev, size_t size,
	dma_addr_t *dma_handle, gfp_t flag)
{
	void *ret = NULL;
	dma_addr_t mapping;
	unsigned int npages, order;
	struct iommu_table *tbl = find_iommu_table(dev);

	size = PAGE_ALIGN(size); /* size rounded up to full pages */
	npages = size >> PAGE_SHIFT;
	order = get_order(size);

	/* alloc enough pages (and possibly more) */
	ret = (void *)__get_free_pages(flag, order);
	if (!ret)
		goto error;
	memset(ret, 0, size);

	if (translation_enabled(tbl)) {
		/* set up tces to cover the allocated range */
		mapping = iommu_alloc(tbl, ret, npages, DMA_BIDIRECTIONAL);
		if (mapping == bad_dma_address)
			goto free;

		*dma_handle = mapping;
	} else /* non translated slot */
		*dma_handle = virt_to_bus(ret);

	return ret;

free: