sba_iommu.c

linux 内核源代码

	*/
	for (index = 0 ; index < (ioc->pdir_size / PDIR_ENTRY_SIZE) ; index++)
		((u64 *)ioc->pdir_base)[index] = (0x80000000000000FF | prefetch_spill_page);
#endif

	/* Clear I/O TLB of any possible entries */
	WRITE_REG(ioc->ibase | (get_iovp_order(ioc->iov_size) + iovp_shift), ioc->ioc_hpa + IOC_PCOM);
	READ_REG(ioc->ioc_hpa + IOC_PCOM);

	/* Enable IOVA translation */
	WRITE_REG(ioc->ibase | 1, ioc->ioc_hpa + IOC_IBASE);
	READ_REG(ioc->ioc_hpa + IOC_IBASE);
}

static void __init
ioc_resource_init(struct ioc *ioc)
{
	spin_lock_init(&ioc->res_lock);
#if DELAYED_RESOURCE_CNT > 0
	spin_lock_init(&ioc->saved_lock);
#endif

	/* resource map size dictated by pdir_size */
	ioc->res_size = ioc->pdir_size / PDIR_ENTRY_SIZE; /* entries */
	ioc->res_size >>= 3;  /* convert bit count to byte count */
	DBG_INIT("%s() res_size 0x%x\n", __FUNCTION__, ioc->res_size);

	ioc->res_map = (char *) __get_free_pages(GFP_KERNEL,
						 get_order(ioc->res_size));
	if (!ioc->res_map)
		panic(PFX "Couldn't allocate resource map\n");

	memset(ioc->res_map, 0, ioc->res_size);
	/* next available IOVP - circular search */
	ioc->res_hint = (unsigned long *) ioc->res_map;

#ifdef ASSERT_PDIR_SANITY
	/* Mark first bit busy - ie no IOVA 0 */
	ioc->res_map[0] = 0x1;
	ioc->pdir_base[0] = 0x8000000000000000ULL | ZX1_SBA_IOMMU_COOKIE;
#endif
#ifdef FULL_VALID_PDIR
	/* Mark the last resource used so we don't prefetch beyond IOVA space */
	ioc->res_map[ioc->res_size - 1] |= 0x80UL; /* res_map is chars */
	ioc->pdir_base[(ioc->pdir_size / PDIR_ENTRY_SIZE) - 1] = (0x80000000000000FF
							      | prefetch_spill_page);
#endif

	DBG_INIT("%s() res_map %x %p\n", __FUNCTION__,
		 ioc->res_size, (void *) ioc->res_map);
}

static void __init
ioc_sac_init(struct ioc *ioc)
{
	struct pci_dev *sac = NULL;
	struct pci_controller *controller = NULL;

	/*
	 * pci_alloc_coherent() must return a DMA address which is
	 * SAC (single address cycle) addressable, so allocate a
	 * pseudo-device to enforce that.
	 */
	sac = kzalloc(sizeof(*sac), GFP_KERNEL);
	if (!sac)
		panic(PFX "Couldn't allocate struct pci_dev");

	controller = kzalloc(sizeof(*controller), GFP_KERNEL);
	if (!controller)
		panic(PFX "Couldn't allocate struct pci_controller");

	controller->iommu = ioc;
	sac->sysdata = controller;
	sac->dma_mask = 0xFFFFFFFFUL;
#ifdef CONFIG_PCI
	sac->dev.bus = &pci_bus_type;
#endif
	ioc->sac_only_dev = sac;
}

static void __init
ioc_zx1_init(struct ioc *ioc)
{
	unsigned long rope_config;
	unsigned int i;

	if (ioc->rev < 0x20)
		panic(PFX "IOC 2.0 or later required for IOMMU support\n");

	/* 38 bit memory controller + extra bit for range displaced by MMIO */
	ioc->dma_mask = (0x1UL << 39) - 1;

	/*
	** Clear ROPE(N)_CONFIG AO bit.
	** Disables "NT Ordering" (~= !"Relaxed Ordering")
	** Overrides bit 1 in DMA Hint Sets.
	** Improves netperf UDP_STREAM by ~10% for tg3 on bcm5701.
	*/
	for (i=0; i<(8*8); i+=8) {
		rope_config = READ_REG(ioc->ioc_hpa + IOC_ROPE0_CFG + i);
		rope_config &= ~IOC_ROPE_AO;
		WRITE_REG(rope_config, ioc->ioc_hpa + IOC_ROPE0_CFG + i);
	}
}

typedef void (initfunc)(struct ioc *);

struct ioc_iommu {
	u32 func_id;
	char *name;
	initfunc *init;
};

static struct ioc_iommu ioc_iommu_info[] __initdata = {
	{ ZX1_IOC_ID, "zx1", ioc_zx1_init },
	{ ZX2_IOC_ID, "zx2", NULL },
	{ SX1000_IOC_ID, "sx1000", NULL },
	{ SX2000_IOC_ID, "sx2000", NULL },
};

static struct ioc * __init
ioc_init(u64 hpa, void *handle)
{
	struct ioc *ioc;
	struct ioc_iommu *info;

	ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);
	if (!ioc)
		return NULL;

	ioc->next = ioc_list;
	ioc_list = ioc;

	ioc->handle = handle;
	ioc->ioc_hpa = ioremap(hpa, 0x1000);

	ioc->func_id = READ_REG(ioc->ioc_hpa + IOC_FUNC_ID);
	ioc->rev = READ_REG(ioc->ioc_hpa + IOC_FCLASS) & 0xFFUL;
	ioc->dma_mask = 0xFFFFFFFFFFFFFFFFUL;	/* conservative */

	for (info = ioc_iommu_info; info < ioc_iommu_info + ARRAY_SIZE(ioc_iommu_info); info++) {
		if (ioc->func_id == info->func_id) {
			ioc->name = info->name;
			if (info->init)
				(info->init)(ioc);
		}
	}

	iovp_size = (1 << iovp_shift);
	iovp_mask = ~(iovp_size - 1);

	DBG_INIT("%s: PAGE_SIZE %ldK, iovp_size %ldK\n", __FUNCTION__,
		PAGE_SIZE >> 10, iovp_size >> 10);

	if (!ioc->name) {
		ioc->name = kmalloc(24, GFP_KERNEL);
		if (ioc->name)
			sprintf((char *) ioc->name, "Unknown (%04x:%04x)",
				ioc->func_id & 0xFFFF, (ioc->func_id >> 16) & 0xFFFF);
		else
			ioc->name = "Unknown";
	}

	ioc_iova_init(ioc);
	ioc_resource_init(ioc);
	ioc_sac_init(ioc);

	if ((long) ~iovp_mask > (long) ia64_max_iommu_merge_mask)
		ia64_max_iommu_merge_mask = ~iovp_mask;

	printk(KERN_INFO PFX
		"%s %d.%d HPA 0x%lx IOVA space %dMb at 0x%lx\n",
		ioc->name, (ioc->rev >> 4) & 0xF, ioc->rev & 0xF,
		hpa, ioc->iov_size >> 20, ioc->ibase);

	return ioc;
}



/**************************************************************************
**
**   SBA initialization code (HW and SW)
**
**   o identify SBA chip itself
**   o FIXME: initialize DMA hints for reasonable defaults
**
**************************************************************************/

#ifdef CONFIG_PROC_FS
static void *
ioc_start(struct seq_file *s, loff_t *pos)
{
	struct ioc *ioc;
	loff_t n = *pos;

	for (ioc = ioc_list; ioc; ioc = ioc->next)
		if (!n--)
			return ioc;

	return NULL;
}

static void *
ioc_next(struct seq_file *s, void *v, loff_t *pos)
{
	struct ioc *ioc = v;

	++*pos;
	return ioc->next;
}

static void
ioc_stop(struct seq_file *s, void *v)
{
}

static int
ioc_show(struct seq_file *s, void *v)
{
	struct ioc *ioc = v;
	unsigned long *res_ptr = (unsigned long *)ioc->res_map;
	int i, used = 0;

	seq_printf(s, "Hewlett Packard %s IOC rev %d.%d\n",
		ioc->name, ((ioc->rev >> 4) & 0xF), (ioc->rev & 0xF));
#ifdef CONFIG_NUMA
	if (ioc->node != MAX_NUMNODES)
		seq_printf(s, "NUMA node       : %d\n", ioc->node);
#endif
	seq_printf(s, "IOVA size       : %ld MB\n", ((ioc->pdir_size >> 3) * iovp_size)/(1024*1024));
	seq_printf(s, "IOVA page size  : %ld kb\n", iovp_size/1024);

	for (i = 0; i < (ioc->res_size / sizeof(unsigned long)); ++i, ++res_ptr)
		used += hweight64(*res_ptr);

	seq_printf(s, "PDIR size       : %d entries\n", ioc->pdir_size >> 3);
	seq_printf(s, "PDIR used       : %d entries\n", used);

#ifdef PDIR_SEARCH_TIMING
	{
		unsigned long i = 0, avg = 0, min, max;
		min = max = ioc->avg_search[0];
		for (i = 0; i < SBA_SEARCH_SAMPLE; i++) {
			avg += ioc->avg_search[i];
			if (ioc->avg_search[i] > max) max = ioc->avg_search[i];
			if (ioc->avg_search[i] < min) min = ioc->avg_search[i];
		}
		avg /= SBA_SEARCH_SAMPLE;
		seq_printf(s, "Bitmap search   : %ld/%ld/%ld (min/avg/max CPU Cycles/IOVA page)\n",
		           min, avg, max);
	}
#endif
#ifndef ALLOW_IOV_BYPASS
	 seq_printf(s, "IOVA bypass disabled\n");
#endif
	return 0;
}

static struct seq_operations ioc_seq_ops = {
	.start = ioc_start,
	.next  = ioc_next,
	.stop  = ioc_stop,
	.show  = ioc_show
};

static int
ioc_open(struct inode *inode, struct file *file)
{
	return seq_open(file, &ioc_seq_ops);
}

static const struct file_operations ioc_fops = {
	.open    = ioc_open,
	.read    = seq_read,
	.llseek  = seq_lseek,
	.release = seq_release
};

static void __init
ioc_proc_init(void)
{
	struct proc_dir_entry *dir, *entry;

	dir = proc_mkdir("bus/mckinley", NULL);
	if (!dir)
		return;

	entry = create_proc_entry(ioc_list->name, 0, dir);
	if (entry)
		entry->proc_fops = &ioc_fops;
}
#endif

static void
sba_connect_bus(struct pci_bus *bus)
{
	acpi_handle handle, parent;
	acpi_status status;
	struct ioc *ioc;

	if (!PCI_CONTROLLER(bus))
		panic(PFX "no sysdata on bus %d!\n", bus->number);

	if (PCI_CONTROLLER(bus)->iommu)
		return;

	handle = PCI_CONTROLLER(bus)->acpi_handle;
	if (!handle)
		return;

	/*
	 * The IOC scope encloses PCI root bridges in the ACPI
	 * namespace, so work our way out until we find an IOC we
	 * claimed previously.
	 */
	do {
		for (ioc = ioc_list; ioc; ioc = ioc->next)
			if (ioc->handle == handle) {
				PCI_CONTROLLER(bus)->iommu = ioc;
				return;
			}

		status = acpi_get_parent(handle, &parent);
		handle = parent;
	} while (ACPI_SUCCESS(status));

	printk(KERN_WARNING "No IOC for PCI Bus %04x:%02x in ACPI\n", pci_domain_nr(bus), bus->number);
}

#ifdef CONFIG_NUMA
static void __init
sba_map_ioc_to_node(struct ioc *ioc, acpi_handle handle)
{
	unsigned int node;
	int pxm;

	ioc->node = MAX_NUMNODES;

	pxm = acpi_get_pxm(handle);

	if (pxm < 0)
		return;

	node = pxm_to_node(pxm);

	if (node >= MAX_NUMNODES || !node_online(node))
		return;

	ioc->node = node;
	return;
}
#else
#define sba_map_ioc_to_node(ioc, handle)
#endif

static int __init
acpi_sba_ioc_add(struct acpi_device *device)
{
	struct ioc *ioc;
	acpi_status status;
	u64 hpa, length;
	struct acpi_buffer buffer;
	struct acpi_device_info *dev_info;

	status = hp_acpi_csr_space(device->handle, &hpa, &length);
	if (ACPI_FAILURE(status))
		return 1;

	buffer.length = ACPI_ALLOCATE_LOCAL_BUFFER;
	status = acpi_get_object_info(device->handle, &buffer);
	if (ACPI_FAILURE(status))
		return 1;
	dev_info = buffer.pointer;

	/*
	 * For HWP0001, only SBA appears in ACPI namespace.  It encloses the PCI
	 * root bridges, and its CSR space includes the IOC function.
	 */
	if (strncmp("HWP0001", dev_info->hardware_id.value, 7) == 0) {
		hpa += ZX1_IOC_OFFSET;
		/* zx1 based systems default to kernel page size iommu pages */
		if (!iovp_shift)
			iovp_shift = min(PAGE_SHIFT, 16);
	}
	kfree(dev_info);

	/*
	 * default anything not caught above or specified on cmdline to 4k
	 * iommu page size
	 */
	if (!iovp_shift)
		iovp_shift = 12;

	ioc = ioc_init(hpa, device->handle);
	if (!ioc)
		return 1;

	/* setup NUMA node association */
	sba_map_ioc_to_node(ioc, device->handle);
	return 0;
}

static const struct acpi_device_id hp_ioc_iommu_device_ids[] = {
	{"HWP0001", 0},
	{"HWP0004", 0},
	{"", 0},
};
static struct acpi_driver acpi_sba_ioc_driver = {
	.name		= "IOC IOMMU Driver",
	.ids		= hp_ioc_iommu_device_ids,
	.ops		= {
		.add	= acpi_sba_ioc_add,
	},
};

static int __init
sba_init(void)
{
	if (!ia64_platform_is("hpzx1") && !ia64_platform_is("hpzx1_swiotlb"))
		return 0;

#if defined(CONFIG_IA64_GENERIC) && defined(CONFIG_CRASH_DUMP) && \
        defined(CONFIG_PROC_FS)
	/* If we are booting a kdump kernel, the sba_iommu will
	 * cause devices that were not shutdown properly to MCA
	 * as soon as they are turned back on.  Our only option for
	 * a successful kdump kernel boot is to use the swiotlb.
	 */
	if (elfcorehdr_addr < ELFCORE_ADDR_MAX) {
		if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
			panic("Unable to initialize software I/O TLB:"
				  " Try machvec=dig boot option");
		machvec_init("dig");
		return 0;
	}
#endif

	acpi_bus_register_driver(&acpi_sba_ioc_driver);
	if (!ioc_list) {
#ifdef CONFIG_IA64_GENERIC
		/*
		 * If we didn't find something sba_iommu can claim, we
		 * need to setup the swiotlb and switch to the dig machvec.
		 */
		if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
			panic("Unable to find SBA IOMMU or initialize "
			      "software I/O TLB: Try machvec=dig boot option");
		machvec_init("dig");
#else
		panic("Unable to find SBA IOMMU: Try a generic or DIG kernel");
#endif
		return 0;
	}

#if defined(CONFIG_IA64_GENERIC) || defined(CONFIG_IA64_HP_ZX1_SWIOTLB)
	/*
	 * hpzx1_swiotlb needs to have a fairly small swiotlb bounce
	 * buffer setup to support devices with smaller DMA masks than
	 * sba_iommu can handle.
	 */
	if (ia64_platform_is("hpzx1_swiotlb")) {
		extern void hwsw_init(void);

		hwsw_init();
	}
#endif

#ifdef CONFIG_PCI
	{
		struct pci_bus *b = NULL;
		while ((b = pci_find_next_bus(b)) != NULL)
			sba_connect_bus(b);
	}
#endif

#ifdef CONFIG_PROC_FS
	ioc_proc_init();
#endif
	return 0;
}

subsys_initcall(sba_init); /* must be initialized after ACPI etc., but before any drivers... */

static int __init
nosbagart(char *str)
{
	reserve_sba_gart = 0;
	return 1;
}

int
sba_dma_supported (struct device *dev, u64 mask)
{
	/* make sure it's at least 32bit capable */
	return ((mask & 0xFFFFFFFFUL) == 0xFFFFFFFFUL);
}

int
sba_dma_mapping_error (dma_addr_t dma_addr)
{
	return 0;
}

__setup("nosbagart", nosbagart);

static int __init
sba_page_override(char *str)
{
	unsigned long page_size;

	page_size = memparse(str, &str);
	switch (page_size) {
		case 4096:
		case 8192:
		case 16384:
		case 65536:
			iovp_shift = ffs(page_size) - 1;
			break;
		default:
			printk("%s: unknown/unsupported iommu page size %ld\n",
			       __FUNCTION__, page_size);
	}

	return 1;
}

__setup("sbapagesize=",sba_page_override);

EXPORT_SYMBOL(sba_dma_mapping_error);
EXPORT_SYMBOL(sba_map_single);
EXPORT_SYMBOL(sba_unmap_single);
EXPORT_SYMBOL(sba_map_sg);
EXPORT_SYMBOL(sba_unmap_sg);
EXPORT_SYMBOL(sba_dma_supported);
EXPORT_SYMBOL(sba_alloc_coherent);
EXPORT_SYMBOL(sba_free_coherent);