intel-iommu.c

linux 内核源代码

	iommu_prepare_gfx_mapping();

	iommu_prepare_isa();

	/*
	 * for each drhd
	 *   enable fault log
	 *   global invalidate context cache
	 *   global invalidate iotlb
	 *   enable translation
	 */
	for_each_drhd_unit(drhd) {
		if (drhd->ignored)
			continue;
		iommu = drhd->iommu;
		sprintf (iommu->name, "dmar%d", unit++);

		iommu_flush_write_buffer(iommu);

		ret = dmar_set_interrupt(iommu);
		if (ret)
			goto error;

		iommu_set_root_entry(iommu);

		iommu_flush_context_global(iommu, 0);
		iommu_flush_iotlb_global(iommu, 0);

		ret = iommu_enable_translation(iommu);
		if (ret)
			goto error;
	}

	return 0;
error:
	for_each_drhd_unit(drhd) {
		if (drhd->ignored)
			continue;
		iommu = drhd->iommu;
		free_iommu(iommu);
	}
	return ret;
}

static inline u64 aligned_size(u64 host_addr, size_t size)
{
	u64 addr;
	addr = (host_addr & (~PAGE_MASK_4K)) + size;
	return PAGE_ALIGN_4K(addr);
}

struct iova *
iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
{
	struct iova *piova;

	/* Make sure it's in range */
	end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
	if (!size || (IOVA_START_ADDR + size > end))
		return NULL;

	piova = alloc_iova(&domain->iovad,
			size >> PAGE_SHIFT_4K, IOVA_PFN(end), 1);
	return piova;
}

static struct iova *
__intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
		size_t size)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct iova *iova = NULL;

	if ((pdev->dma_mask <= DMA_32BIT_MASK) || (dmar_forcedac)) {
		iova = iommu_alloc_iova(domain, size, pdev->dma_mask);
	} else  {
		/*
		 * First try to allocate an io virtual address in
		 * DMA_32BIT_MASK and if that fails then try allocating
		 * from higer range
		 */
		iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
		if (!iova)
			iova = iommu_alloc_iova(domain, size, pdev->dma_mask);
	}

	if (!iova) {
		printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
		return NULL;
	}

	return iova;
}

static struct dmar_domain *
get_valid_domain_for_dev(struct pci_dev *pdev)
{
	struct dmar_domain *domain;
	int ret;

	domain = get_domain_for_dev(pdev,
			DEFAULT_DOMAIN_ADDRESS_WIDTH);
	if (!domain) {
		printk(KERN_ERR
			"Allocating domain for %s failed", pci_name(pdev));
		return NULL;
	}

	/* make sure context mapping is ok */
	if (unlikely(!domain_context_mapped(domain, pdev))) {
		ret = domain_context_mapping(domain, pdev);
		if (ret) {
			printk(KERN_ERR
				"Domain context map for %s failed",
				pci_name(pdev));
			return NULL;
		}
	}

	return domain;
}

static dma_addr_t intel_map_single(struct device *hwdev, void *addr,
	size_t size, int dir)
{
	struct pci_dev *pdev = to_pci_dev(hwdev);
	int ret;
	struct dmar_domain *domain;
	unsigned long start_addr;
	struct iova *iova;
	int prot = 0;

	BUG_ON(dir == DMA_NONE);
	if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
		return virt_to_bus(addr);

	domain = get_valid_domain_for_dev(pdev);
	if (!domain)
		return 0;

	addr = (void *)virt_to_phys(addr);
	size = aligned_size((u64)addr, size);

	iova = __intel_alloc_iova(hwdev, domain, size);
	if (!iova)
		goto error;

	start_addr = iova->pfn_lo << PAGE_SHIFT_4K;

	/*
	 * Check if DMAR supports zero-length reads on write only
	 * mappings..
	 */
	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
			!cap_zlr(domain->iommu->cap))
		prot |= DMA_PTE_READ;
	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
		prot |= DMA_PTE_WRITE;
	/*
	 * addr - (addr + size) might be partial page, we should map the whole
	 * page.  Note: if two part of one page are separately mapped, we
	 * might have two guest_addr mapping to the same host addr, but this
	 * is not a big problem
	 */
	ret = domain_page_mapping(domain, start_addr,
		((u64)addr) & PAGE_MASK_4K, size, prot);
	if (ret)
		goto error;

	pr_debug("Device %s request: %lx@%llx mapping: %lx@%llx, dir %d\n",
		pci_name(pdev), size, (u64)addr,
		size, (u64)start_addr, dir);

	/* it's a non-present to present mapping */
	ret = iommu_flush_iotlb_psi(domain->iommu, domain->id,
			start_addr, size >> PAGE_SHIFT_4K, 1);
	if (ret)
		iommu_flush_write_buffer(domain->iommu);

	return (start_addr + ((u64)addr & (~PAGE_MASK_4K)));

error:
	if (iova)
		__free_iova(&domain->iovad, iova);
	printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
		pci_name(pdev), size, (u64)addr, dir);
	return 0;
}

static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr,
	size_t size, int dir)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct dmar_domain *domain;
	unsigned long start_addr;
	struct iova *iova;

	if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
		return;
	domain = find_domain(pdev);
	BUG_ON(!domain);

	iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
	if (!iova)
		return;

	start_addr = iova->pfn_lo << PAGE_SHIFT_4K;
	size = aligned_size((u64)dev_addr, size);

	pr_debug("Device %s unmapping: %lx@%llx\n",
		pci_name(pdev), size, (u64)start_addr);

	/*  clear the whole page */
	dma_pte_clear_range(domain, start_addr, start_addr + size);
	/* free page tables */
	dma_pte_free_pagetable(domain, start_addr, start_addr + size);

	if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
			size >> PAGE_SHIFT_4K, 0))
		iommu_flush_write_buffer(domain->iommu);

	/* free iova */
	__free_iova(&domain->iovad, iova);
}

static void * intel_alloc_coherent(struct device *hwdev, size_t size,
		       dma_addr_t *dma_handle, gfp_t flags)
{
	void *vaddr;
	int order;

	size = PAGE_ALIGN_4K(size);
	order = get_order(size);
	flags &= ~(GFP_DMA | GFP_DMA32);

	vaddr = (void *)__get_free_pages(flags, order);
	if (!vaddr)
		return NULL;
	memset(vaddr, 0, size);

	*dma_handle = intel_map_single(hwdev, vaddr, size, DMA_BIDIRECTIONAL);
	if (*dma_handle)
		return vaddr;
	free_pages((unsigned long)vaddr, order);
	return NULL;
}

static void intel_free_coherent(struct device *hwdev, size_t size,
	void *vaddr, dma_addr_t dma_handle)
{
	int order;

	size = PAGE_ALIGN_4K(size);
	order = get_order(size);

	intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
	free_pages((unsigned long)vaddr, order);
}

#define SG_ENT_VIRT_ADDRESS(sg)	(sg_virt((sg)))
static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
	int nelems, int dir)
{
	int i;
	struct pci_dev *pdev = to_pci_dev(hwdev);
	struct dmar_domain *domain;
	unsigned long start_addr;
	struct iova *iova;
	size_t size = 0;
	void *addr;
	struct scatterlist *sg;

	if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
		return;

	domain = find_domain(pdev);

	iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
	if (!iova)
		return;
	for_each_sg(sglist, sg, nelems, i) {
		addr = SG_ENT_VIRT_ADDRESS(sg);
		size += aligned_size((u64)addr, sg->length);
	}

	start_addr = iova->pfn_lo << PAGE_SHIFT_4K;

	/*  clear the whole page */
	dma_pte_clear_range(domain, start_addr, start_addr + size);
	/* free page tables */
	dma_pte_free_pagetable(domain, start_addr, start_addr + size);

	if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
			size >> PAGE_SHIFT_4K, 0))
		iommu_flush_write_buffer(domain->iommu);

	/* free iova */
	__free_iova(&domain->iovad, iova);
}

static int intel_nontranslate_map_sg(struct device *hddev,
	struct scatterlist *sglist, int nelems, int dir)
{
	int i;
	struct scatterlist *sg;

	for_each_sg(sglist, sg, nelems, i) {
		BUG_ON(!sg_page(sg));
		sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
		sg->dma_length = sg->length;
	}
	return nelems;
}

static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist,
				int nelems, int dir)
{
	void *addr;
	int i;
	struct pci_dev *pdev = to_pci_dev(hwdev);
	struct dmar_domain *domain;
	size_t size = 0;
	int prot = 0;
	size_t offset = 0;
	struct iova *iova = NULL;
	int ret;
	struct scatterlist *sg;
	unsigned long start_addr;

	BUG_ON(dir == DMA_NONE);
	if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
		return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);

	domain = get_valid_domain_for_dev(pdev);
	if (!domain)
		return 0;

	for_each_sg(sglist, sg, nelems, i) {
		addr = SG_ENT_VIRT_ADDRESS(sg);
		addr = (void *)virt_to_phys(addr);
		size += aligned_size((u64)addr, sg->length);
	}

	iova = __intel_alloc_iova(hwdev, domain, size);
	if (!iova) {
		sglist->dma_length = 0;
		return 0;
	}

	/*
	 * Check if DMAR supports zero-length reads on write only
	 * mappings..
	 */
	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
			!cap_zlr(domain->iommu->cap))
		prot |= DMA_PTE_READ;
	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
		prot |= DMA_PTE_WRITE;

	start_addr = iova->pfn_lo << PAGE_SHIFT_4K;
	offset = 0;
	for_each_sg(sglist, sg, nelems, i) {
		addr = SG_ENT_VIRT_ADDRESS(sg);
		addr = (void *)virt_to_phys(addr);
		size = aligned_size((u64)addr, sg->length);
		ret = domain_page_mapping(domain, start_addr + offset,
			((u64)addr) & PAGE_MASK_4K,
			size, prot);
		if (ret) {
			/*  clear the page */
			dma_pte_clear_range(domain, start_addr,
				  start_addr + offset);
			/* free page tables */
			dma_pte_free_pagetable(domain, start_addr,
				  start_addr + offset);
			/* free iova */
			__free_iova(&domain->iovad, iova);
			return 0;
		}
		sg->dma_address = start_addr + offset +
				((u64)addr & (~PAGE_MASK_4K));
		sg->dma_length = sg->length;
		offset += size;
	}

	/* it's a non-present to present mapping */
	if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
			start_addr, offset >> PAGE_SHIFT_4K, 1))
		iommu_flush_write_buffer(domain->iommu);
	return nelems;
}

static struct dma_mapping_ops intel_dma_ops = {
	.alloc_coherent = intel_alloc_coherent,
	.free_coherent = intel_free_coherent,
	.map_single = intel_map_single,
	.unmap_single = intel_unmap_single,
	.map_sg = intel_map_sg,
	.unmap_sg = intel_unmap_sg,
};

static inline int iommu_domain_cache_init(void)
{
	int ret = 0;

	iommu_domain_cache = kmem_cache_create("iommu_domain",
					 sizeof(struct dmar_domain),
					 0,
					 SLAB_HWCACHE_ALIGN,

					 NULL);
	if (!iommu_domain_cache) {
		printk(KERN_ERR "Couldn't create iommu_domain cache\n");
		ret = -ENOMEM;
	}

	return ret;
}

static inline int iommu_devinfo_cache_init(void)
{
	int ret = 0;

	iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
					 sizeof(struct device_domain_info),
					 0,
					 SLAB_HWCACHE_ALIGN,

					 NULL);
	if (!iommu_devinfo_cache) {
		printk(KERN_ERR "Couldn't create devinfo cache\n");
		ret = -ENOMEM;
	}

	return ret;
}

static inline int iommu_iova_cache_init(void)
{
	int ret = 0;

	iommu_iova_cache = kmem_cache_create("iommu_iova",
					 sizeof(struct iova),
					 0,
					 SLAB_HWCACHE_ALIGN,

					 NULL);
	if (!iommu_iova_cache) {
		printk(KERN_ERR "Couldn't create iova cache\n");
		ret = -ENOMEM;
	}

	return ret;
}

static int __init iommu_init_mempool(void)
{
	int ret;
	ret = iommu_iova_cache_init();
	if (ret)
		return ret;

	ret = iommu_domain_cache_init();
	if (ret)
		goto domain_error;

	ret = iommu_devinfo_cache_init();
	if (!ret)
		return ret;

	kmem_cache_destroy(iommu_domain_cache);
domain_error:
	kmem_cache_destroy(iommu_iova_cache);

	return -ENOMEM;
}

static void __init iommu_exit_mempool(void)
{
	kmem_cache_destroy(iommu_devinfo_cache);
	kmem_cache_destroy(iommu_domain_cache);
	kmem_cache_destroy(iommu_iova_cache);

}

void __init detect_intel_iommu(void)
{
	if (swiotlb || no_iommu || iommu_detected || dmar_disabled)
		return;
	if (early_dmar_detect()) {
		iommu_detected = 1;
	}
}

static void __init init_no_remapping_devices(void)
{
	struct dmar_drhd_unit *drhd;

	for_each_drhd_unit(drhd) {
		if (!drhd->include_all) {
			int i;
			for (i = 0; i < drhd->devices_cnt; i++)
				if (drhd->devices[i] != NULL)
					break;
			/* ignore DMAR unit if no pci devices exist */
			if (i == drhd->devices_cnt)
				drhd->ignored = 1;
		}
	}

	if (dmar_map_gfx)
		return;

	for_each_drhd_unit(drhd) {
		int i;
		if (drhd->ignored || drhd->include_all)
			continue;

		for (i = 0; i < drhd->devices_cnt; i++)
			if (drhd->devices[i] &&
				!IS_GFX_DEVICE(drhd->devices[i]))
				break;

		if (i < drhd->devices_cnt)
			continue;

		/* bypass IOMMU if it is just for gfx devices */
		drhd->ignored = 1;
		for (i = 0; i < drhd->devices_cnt; i++) {
			if (!drhd->devices[i])
				continue;
			drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
		}
	}
}

int __init intel_iommu_init(void)
{
	int ret = 0;

	if (no_iommu || swiotlb || dmar_disabled)
		return -ENODEV;

	if (dmar_table_init())
		return 	-ENODEV;

	iommu_init_mempool();
	dmar_init_reserved_ranges();

	init_no_remapping_devices();

	ret = init_dmars();
	if (ret) {
		printk(KERN_ERR "IOMMU: dmar init failed\n");
		put_iova_domain(&reserved_iova_list);
		iommu_exit_mempool();
		return ret;
	}
	printk(KERN_INFO
	"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");

	force_iommu = 1;
	dma_ops = &intel_dma_ops;
	return 0;
}