pci-gart.c

linux-2.4.29操作系统的源码

/*
 * Dynamic DMA mapping support for AMD Hammer.
 * 
 * Use the integrated AGP GART in the Hammer northbridge as an IOMMU for PCI.
 * This allows to use PCI devices that only support 32bit addresses on systems
 * with more than 4GB. 
 *
 * See Documentation/DMA-mapping.txt for the interface specification.
 * 
 * Copyright 2002 Andi Kleen, SuSE Labs.
 * $Id: pci-gart.c,v 1.32 2004/02/27 18:30:19 ak Exp $
 */

#include <linux/config.h>
#include <linux/types.h>
#include <linux/ctype.h>
#include <linux/agp_backend.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/spinlock.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
#include <linux/module.h>
#include <asm/io.h>
#include <asm/mtrr.h>
#include <asm/bitops.h>
#include <asm/pgtable.h>
#include <asm/proto.h>
#include "pci-x86_64.h"

unsigned long iommu_bus_base;	/* GART remapping area (physical) */
static unsigned long iommu_size; 	/* size of remapping area bytes */
static unsigned long iommu_pages;	/* .. and in pages */

u32 *iommu_gatt_base; 		/* Remapping table */

int no_iommu; 
static int no_agp; 
#ifdef CONFIG_IOMMU_DEBUG
int force_mmu = 1;
#else
int force_mmu = 0;
#endif
int iommu_fullflush = 1;

extern int fallback_aper_order;
extern int fallback_aper_force;

/* Allocation bitmap for the remapping area */ 
static spinlock_t iommu_bitmap_lock = SPIN_LOCK_UNLOCKED;
static unsigned long *iommu_gart_bitmap; /* guarded by iommu_bitmap_lock */

#define GPTE_VALID    1
#define GPTE_COHERENT 2
#define GPTE_ENCODE(x) (((x) & 0xfffff000) | (((x) >> 32) << 4) | GPTE_VALID | GPTE_COHERENT)
#define GPTE_DECODE(x) (((x) & 0xfffff000) | (((u64)(x) & 0xff0) << 28))

#define for_all_nb(dev) \
	pci_for_each_dev(dev) \
		if (dev->vendor == PCI_VENDOR_ID_AMD && dev->device==0x1103 &&\
		    dev->bus->number == 0 && PCI_FUNC(dev->devfn) == 3 && \
		    (PCI_SLOT(dev->devfn) >= 24) && (PCI_SLOT(dev->devfn) <= 31))

#define EMERGENCY_PAGES 32 /* = 128KB */ 

#ifdef CONFIG_AGP
extern int agp_init(void);
#define AGPEXTERN extern
#else
#define AGPEXTERN
#endif

/* backdoor interface to AGP driver */
AGPEXTERN int agp_memory_reserved;
AGPEXTERN __u32 *agp_gatt_table;

static unsigned long next_bit;  /* protected by iommu_bitmap_lock */

static struct pci_dev *northbridges[NR_CPUS + 1];
static u32 northbridge_flush_word[NR_CPUS + 1];
static int need_flush; 	/* global flush state. set for each gart wrap */
static unsigned long alloc_iommu(int size) 
{ 	
	unsigned long offset, flags;

	spin_lock_irqsave(&iommu_bitmap_lock, flags);	
	offset = find_next_zero_string(iommu_gart_bitmap,next_bit,iommu_pages,size);
	if (offset == -1) {
		need_flush = 1;
	       	offset = find_next_zero_string(iommu_gart_bitmap,0,next_bit,size);
	}
	if (offset != -1) { 
		set_bit_string(iommu_gart_bitmap, offset, size); 
		next_bit = offset+size; 
		if (next_bit >= iommu_pages) { 
			need_flush = 1;
			next_bit = 0;
	} 
	} 
	if (iommu_fullflush)
		need_flush = 1;
	spin_unlock_irqrestore(&iommu_bitmap_lock, flags);      
	return offset;
} 

static void free_iommu(unsigned long offset, int size)
{ 
	unsigned long flags;
	spin_lock_irqsave(&iommu_bitmap_lock, flags);
	clear_bit_string(iommu_gart_bitmap, offset, size);
	spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
} 


/* 
 * Use global flush state to avoid races with multiple flushers.
 */
static void __flush_gart(void)
{ 
	unsigned long flags;
	int flushed = 0;
	int i;

	spin_lock_irqsave(&iommu_bitmap_lock, flags);
	/* recheck flush count inside lock */
	if (need_flush) { 
		for (i = 0; northbridges[i]; i++) { 
			u32 w;
			pci_write_config_dword(northbridges[i], 0x9c, 
					       northbridge_flush_word[i] | 1); 
			do { 
				pci_read_config_dword(northbridges[i], 0x9c, &w);
			} while (w & 1);
			flushed++;
		} 
		if (!flushed) 
			printk("nothing to flush?\n");
		need_flush = 0;
	} 
	spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
} 

static inline void flush_gart(void)
{ 
	if (need_flush)
		__flush_gart();
} 

void *pci_alloc_consistent(struct pci_dev *hwdev, size_t size,
			   dma_addr_t *dma_handle)
{
	void *memory;
	int gfp = GFP_ATOMIC;
	int i;
	unsigned long iommu_page;

	if (hwdev == NULL || hwdev->dma_mask < 0xffffffff || no_iommu)
		gfp |= GFP_DMA;

	/* 
	 * First try to allocate continuous and use directly if already 
	 * in lowmem. 
	 */ 
	size = round_up(size, PAGE_SIZE); 
	memory = (void *)__get_free_pages(gfp, get_order(size));
	if (memory == NULL) {
		return NULL; 
	} else {
		int high = 0, mmu;
		if (((unsigned long)virt_to_bus(memory) + size) > 0xffffffffUL)
			high = 1;
		mmu = high;
		if (force_mmu && !(gfp & GFP_DMA)) 
			mmu = 1;
		if (no_iommu) { 
			if (high) goto error;
			mmu = 0; 
		} 	
		memset(memory, 0, size); 
		if (!mmu) { 
			*dma_handle = virt_to_bus(memory);
			return memory;
		}
	} 

	size >>= PAGE_SHIFT;

	iommu_page = alloc_iommu(size);
	if (iommu_page == -1)
		goto error; 

   	/* Fill in the GATT, allocating pages as needed. */
	for (i = 0; i < size; i++) { 
		unsigned long phys_mem; 
		void *mem = memory + i*PAGE_SIZE;
		if (i > 0) 
			atomic_inc(&virt_to_page(mem)->count); 
		phys_mem = virt_to_phys(mem); 
		BUG_ON(phys_mem & ~PHYSICAL_PAGE_MASK); 
		iommu_gatt_base[iommu_page + i] = GPTE_ENCODE(phys_mem); 
	} 

	flush_gart();
	*dma_handle = iommu_bus_base + (iommu_page << PAGE_SHIFT);
	return memory; 
	
 error:
	free_pages((unsigned long)memory, get_order(size)); 
	return NULL; 
}

/* 
 * Unmap consistent memory.
 * The caller must ensure that the device has finished accessing the mapping.
 */
void pci_free_consistent(struct pci_dev *hwdev, size_t size,
			 void *vaddr, dma_addr_t bus)
{
	unsigned long iommu_page;

	size = round_up(size, PAGE_SIZE); 
	if (bus >= iommu_bus_base && bus < iommu_bus_base + iommu_size) { 
		unsigned pages = size >> PAGE_SHIFT;
		iommu_page = (bus - iommu_bus_base) >> PAGE_SHIFT;
		vaddr = __va(GPTE_DECODE(iommu_gatt_base[iommu_page]));
		int i;
		for (i = 0; i < pages; i++) {
			u64 pte = iommu_gatt_base[iommu_page + i];
		BUG_ON((pte & GPTE_VALID) == 0); 
		iommu_gatt_base[iommu_page + i] = 0; 		
	} 
		free_iommu(iommu_page, pages);
	}
	free_pages((unsigned long)vaddr, get_order(size)); 		
}

#ifdef CONFIG_IOMMU_LEAK
/* Debugging aid for drivers that don't free their IOMMU tables */
static void **iommu_leak_tab; 
static int leak_trace;
int iommu_leak_pages = 20; 
extern unsigned long printk_address(unsigned long);
void dump_leak(void)
{
	int i;
	static int dump; 
	if (dump || !iommu_leak_tab) return;
	dump = 1;
	show_stack(NULL);
	/* Very crude. dump some from the end of the table too */ 
	printk("Dumping %d pages from end of IOMMU:\n", iommu_leak_pages); 
	for (i = 0; i < iommu_leak_pages; i+=2) {
		printk("%lu: ", iommu_pages-i);
		printk_address((unsigned long) iommu_leak_tab[iommu_pages-i]);
		printk("%c", (i+1)%2 == 0 ? '\n' : ' '); 
	} 
	printk("\n");
}
#endif

static void iommu_full(struct pci_dev *dev, void *addr, size_t size, int dir)
{
	/* 
	 * Ran out of IOMMU space for this operation. This is very bad.
	 * Unfortunately the drivers cannot handle this operation properly.
	 * Return some non mapped prereserved space in the aperture and 
	 * let the Northbridge deal with it. This will result in garbage
	 * in the IO operation. When the size exceeds the prereserved spa	 * memory corruption will occur or random memory will be DMAed 
	 * out. Hopefully no network devices use single mappings that big.
	 */ 
	
	printk(KERN_ERR 
  "PCI-DMA: Error: ran out out IOMMU space for %p size %lu at device %s[%s]\n",
	       addr,size, dev ? dev->name : "?", dev ? dev->slot_name : "?");

	if (size > PAGE_SIZE*EMERGENCY_PAGES) {
		if (dir == PCI_DMA_FROMDEVICE || dir == PCI_DMA_BIDIRECTIONAL)
			panic("PCI-DMA: Memory will be corrupted\n");
		if (dir == PCI_DMA_TODEVICE || dir == PCI_DMA_BIDIRECTIONAL) 
			panic("PCI-DMA: Random memory will be DMAed\n"); 
	} 

#ifdef CONFIG_IOMMU_LEAK
	dump_leak(); 
#endif
} 

static inline int need_iommu(struct pci_dev *dev, unsigned long addr, size_t size)
{ 
	u64 mask = dev ? dev->dma_mask : 0xffffffff;
	int high = (~mask & (unsigned long)(addr + size)) != 0;
	int mmu = high;
	if (force_mmu) 
		mmu = 1; 
	if (no_iommu) { 
		if (high) 
			panic("pci_map_single: high address but no IOMMU.\n"); 
		mmu = 0; 
	} 	
	return mmu; 
}

dma_addr_t pci_map_single(struct pci_dev *dev, void *addr, size_t size,
			    int dir)
{ 
	unsigned long iommu_page;
	unsigned long phys_mem, bus;
	int i, npages;

	BUG_ON(dir == PCI_DMA_NONE);

#ifdef CONFIG_SWIOTLB
       if (swiotlb)
               return swiotlb_map_single(dev,addr,size,dir);
#endif


	phys_mem = virt_to_phys(addr); 
	if (!need_iommu(dev, phys_mem, size))
		return phys_mem; 

	npages = round_up(size + ((u64)addr & ~PAGE_MASK), PAGE_SIZE) >> PAGE_SHIFT;

	iommu_page = alloc_iommu(npages); 
	if (iommu_page == -1) {