intel-iommu.c

linux 内核源代码

	int non_present_entry_flush)
{
	return __iommu_flush_context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
		non_present_entry_flush);
}

static int inline iommu_flush_context_device(struct intel_iommu *iommu,
	u16 did, u16 source_id, u8 function_mask, int non_present_entry_flush)
{
	return __iommu_flush_context(iommu, did, source_id, function_mask,
		DMA_CCMD_DEVICE_INVL, non_present_entry_flush);
}

/* return value determine if we need a write buffer flush */
static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
	u64 addr, unsigned int size_order, u64 type,
	int non_present_entry_flush)
{
	int tlb_offset = ecap_iotlb_offset(iommu->ecap);
	u64 val = 0, val_iva = 0;
	unsigned long flag;

	/*
	 * In the non-present entry flush case, if hardware doesn't cache
	 * non-present entry we do nothing and if hardware cache non-present
	 * entry, we flush entries of domain 0 (the domain id is used to cache
	 * any non-present entries)
	 */
	if (non_present_entry_flush) {
		if (!cap_caching_mode(iommu->cap))
			return 1;
		else
			did = 0;
	}

	switch (type) {
	case DMA_TLB_GLOBAL_FLUSH:
		/* global flush doesn't need set IVA_REG */
		val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
		break;
	case DMA_TLB_DSI_FLUSH:
		val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
		break;
	case DMA_TLB_PSI_FLUSH:
		val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
		/* Note: always flush non-leaf currently */
		val_iva = size_order | addr;
		break;
	default:
		BUG();
	}
	/* Note: set drain read/write */
#if 0
	/*
	 * This is probably to be super secure.. Looks like we can
	 * ignore it without any impact.
	 */
	if (cap_read_drain(iommu->cap))
		val |= DMA_TLB_READ_DRAIN;
#endif
	if (cap_write_drain(iommu->cap))
		val |= DMA_TLB_WRITE_DRAIN;

	spin_lock_irqsave(&iommu->register_lock, flag);
	/* Note: Only uses first TLB reg currently */
	if (val_iva)
		dmar_writeq(iommu->reg + tlb_offset, val_iva);
	dmar_writeq(iommu->reg + tlb_offset + 8, val);

	/* Make sure hardware complete it */
	IOMMU_WAIT_OP(iommu, tlb_offset + 8,
		dmar_readq, (!(val & DMA_TLB_IVT)), val);

	spin_unlock_irqrestore(&iommu->register_lock, flag);

	/* check IOTLB invalidation granularity */
	if (DMA_TLB_IAIG(val) == 0)
		printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
	if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
		pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
			DMA_TLB_IIRG(type), DMA_TLB_IAIG(val));
	/* flush context entry will implictly flush write buffer */
	return 0;
}

static int inline iommu_flush_iotlb_global(struct intel_iommu *iommu,
	int non_present_entry_flush)
{
	return __iommu_flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
		non_present_entry_flush);
}

static int inline iommu_flush_iotlb_dsi(struct intel_iommu *iommu, u16 did,
	int non_present_entry_flush)
{
	return __iommu_flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
		non_present_entry_flush);
}

static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
	u64 addr, unsigned int pages, int non_present_entry_flush)
{
	unsigned int mask;

	BUG_ON(addr & (~PAGE_MASK_4K));
	BUG_ON(pages == 0);

	/* Fallback to domain selective flush if no PSI support */
	if (!cap_pgsel_inv(iommu->cap))
		return iommu_flush_iotlb_dsi(iommu, did,
			non_present_entry_flush);

	/*
	 * PSI requires page size to be 2 ^ x, and the base address is naturally
	 * aligned to the size
	 */
	mask = ilog2(__roundup_pow_of_two(pages));
	/* Fallback to domain selective flush if size is too big */
	if (mask > cap_max_amask_val(iommu->cap))
		return iommu_flush_iotlb_dsi(iommu, did,
			non_present_entry_flush);

	return __iommu_flush_iotlb(iommu, did, addr, mask,
		DMA_TLB_PSI_FLUSH, non_present_entry_flush);
}

static int iommu_enable_translation(struct intel_iommu *iommu)
{
	u32 sts;
	unsigned long flags;

	spin_lock_irqsave(&iommu->register_lock, flags);
	writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);

	/* Make sure hardware complete it */
	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
		readl, (sts & DMA_GSTS_TES), sts);

	iommu->gcmd |= DMA_GCMD_TE;
	spin_unlock_irqrestore(&iommu->register_lock, flags);
	return 0;
}

static int iommu_disable_translation(struct intel_iommu *iommu)
{
	u32 sts;
	unsigned long flag;

	spin_lock_irqsave(&iommu->register_lock, flag);
	iommu->gcmd &= ~DMA_GCMD_TE;
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

	/* Make sure hardware complete it */
	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
		readl, (!(sts & DMA_GSTS_TES)), sts);

	spin_unlock_irqrestore(&iommu->register_lock, flag);
	return 0;
}

/* iommu interrupt handling. Most stuff are MSI-like. */

static char *fault_reason_strings[] =
{
	"Software",
	"Present bit in root entry is clear",
	"Present bit in context entry is clear",
	"Invalid context entry",
	"Access beyond MGAW",
	"PTE Write access is not set",
	"PTE Read access is not set",
	"Next page table ptr is invalid",
	"Root table address invalid",
	"Context table ptr is invalid",
	"non-zero reserved fields in RTP",
	"non-zero reserved fields in CTP",
	"non-zero reserved fields in PTE",
	"Unknown"
};
#define MAX_FAULT_REASON_IDX 	ARRAY_SIZE(fault_reason_strings) - 1

char *dmar_get_fault_reason(u8 fault_reason)
{
	if (fault_reason >= MAX_FAULT_REASON_IDX)
		return fault_reason_strings[MAX_FAULT_REASON_IDX - 1];
	else
		return fault_reason_strings[fault_reason];
}

void dmar_msi_unmask(unsigned int irq)
{
	struct intel_iommu *iommu = get_irq_data(irq);
	unsigned long flag;

	/* unmask it */
	spin_lock_irqsave(&iommu->register_lock, flag);
	writel(0, iommu->reg + DMAR_FECTL_REG);
	/* Read a reg to force flush the post write */
	readl(iommu->reg + DMAR_FECTL_REG);
	spin_unlock_irqrestore(&iommu->register_lock, flag);
}

void dmar_msi_mask(unsigned int irq)
{
	unsigned long flag;
	struct intel_iommu *iommu = get_irq_data(irq);

	/* mask it */
	spin_lock_irqsave(&iommu->register_lock, flag);
	writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
	/* Read a reg to force flush the post write */
	readl(iommu->reg + DMAR_FECTL_REG);
	spin_unlock_irqrestore(&iommu->register_lock, flag);
}

void dmar_msi_write(int irq, struct msi_msg *msg)
{
	struct intel_iommu *iommu = get_irq_data(irq);
	unsigned long flag;

	spin_lock_irqsave(&iommu->register_lock, flag);
	writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
	writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
	writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
	spin_unlock_irqrestore(&iommu->register_lock, flag);
}

void dmar_msi_read(int irq, struct msi_msg *msg)
{
	struct intel_iommu *iommu = get_irq_data(irq);
	unsigned long flag;

	spin_lock_irqsave(&iommu->register_lock, flag);
	msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
	msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
	msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
	spin_unlock_irqrestore(&iommu->register_lock, flag);
}

static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
		u8 fault_reason, u16 source_id, u64 addr)
{
	char *reason;

	reason = dmar_get_fault_reason(fault_reason);

	printk(KERN_ERR
		"DMAR:[%s] Request device [%02x:%02x.%d] "
		"fault addr %llx \n"
		"DMAR:[fault reason %02d] %s\n",
		(type ? "DMA Read" : "DMA Write"),
		(source_id >> 8), PCI_SLOT(source_id & 0xFF),
		PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
	return 0;
}

#define PRIMARY_FAULT_REG_LEN (16)
static irqreturn_t iommu_page_fault(int irq, void *dev_id)
{
	struct intel_iommu *iommu = dev_id;
	int reg, fault_index;
	u32 fault_status;
	unsigned long flag;

	spin_lock_irqsave(&iommu->register_lock, flag);
	fault_status = readl(iommu->reg + DMAR_FSTS_REG);

	/* TBD: ignore advanced fault log currently */
	if (!(fault_status & DMA_FSTS_PPF))
		goto clear_overflow;

	fault_index = dma_fsts_fault_record_index(fault_status);
	reg = cap_fault_reg_offset(iommu->cap);
	while (1) {
		u8 fault_reason;
		u16 source_id;
		u64 guest_addr;
		int type;
		u32 data;

		/* highest 32 bits */
		data = readl(iommu->reg + reg +
				fault_index * PRIMARY_FAULT_REG_LEN + 12);
		if (!(data & DMA_FRCD_F))
			break;

		fault_reason = dma_frcd_fault_reason(data);
		type = dma_frcd_type(data);

		data = readl(iommu->reg + reg +
				fault_index * PRIMARY_FAULT_REG_LEN + 8);
		source_id = dma_frcd_source_id(data);

		guest_addr = dmar_readq(iommu->reg + reg +
				fault_index * PRIMARY_FAULT_REG_LEN);
		guest_addr = dma_frcd_page_addr(guest_addr);
		/* clear the fault */
		writel(DMA_FRCD_F, iommu->reg + reg +
			fault_index * PRIMARY_FAULT_REG_LEN + 12);

		spin_unlock_irqrestore(&iommu->register_lock, flag);

		iommu_page_fault_do_one(iommu, type, fault_reason,
				source_id, guest_addr);

		fault_index++;
		if (fault_index > cap_num_fault_regs(iommu->cap))
			fault_index = 0;
		spin_lock_irqsave(&iommu->register_lock, flag);
	}
clear_overflow:
	/* clear primary fault overflow */
	fault_status = readl(iommu->reg + DMAR_FSTS_REG);
	if (fault_status & DMA_FSTS_PFO)
		writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);

	spin_unlock_irqrestore(&iommu->register_lock, flag);
	return IRQ_HANDLED;
}

int dmar_set_interrupt(struct intel_iommu *iommu)
{
	int irq, ret;

	irq = create_irq();
	if (!irq) {
		printk(KERN_ERR "IOMMU: no free vectors\n");
		return -EINVAL;
	}

	set_irq_data(irq, iommu);
	iommu->irq = irq;

	ret = arch_setup_dmar_msi(irq);
	if (ret) {
		set_irq_data(irq, NULL);
		iommu->irq = 0;
		destroy_irq(irq);
		return 0;
	}

	/* Force fault register is cleared */
	iommu_page_fault(irq, iommu);

	ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
	if (ret)
		printk(KERN_ERR "IOMMU: can't request irq\n");
	return ret;
}

static int iommu_init_domains(struct intel_iommu *iommu)
{
	unsigned long ndomains;
	unsigned long nlongs;

	ndomains = cap_ndoms(iommu->cap);
	pr_debug("Number of Domains supportd <%ld>\n", ndomains);
	nlongs = BITS_TO_LONGS(ndomains);

	/* TBD: there might be 64K domains,
	 * consider other allocation for future chip
	 */
	iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
	if (!iommu->domain_ids) {
		printk(KERN_ERR "Allocating domain id array failed\n");
		return -ENOMEM;
	}
	iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
			GFP_KERNEL);
	if (!iommu->domains) {
		printk(KERN_ERR "Allocating domain array failed\n");
		kfree(iommu->domain_ids);
		return -ENOMEM;
	}

	/*
	 * if Caching mode is set, then invalid translations are tagged
	 * with domainid 0. Hence we need to pre-allocate it.
	 */
	if (cap_caching_mode(iommu->cap))
		set_bit(0, iommu->domain_ids);
	return 0;
}

static struct intel_iommu *alloc_iommu(struct dmar_drhd_unit *drhd)
{
	struct intel_iommu *iommu;
	int ret;
	int map_size;
	u32 ver;

	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
	if (!iommu)
		return NULL;
	iommu->reg = ioremap(drhd->reg_base_addr, PAGE_SIZE_4K);
	if (!iommu->reg) {
		printk(KERN_ERR "IOMMU: can't map the region\n");
		goto error;
	}
	iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
	iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);

	/* the registers might be more than one page */
	map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
		cap_max_fault_reg_offset(iommu->cap));
	map_size = PAGE_ALIGN_4K(map_size);
	if (map_size > PAGE_SIZE_4K) {
		iounmap(iommu->reg);
		iommu->reg = ioremap(drhd->reg_base_addr, map_size);
		if (!iommu->reg) {
			printk(KERN_ERR "IOMMU: can't map the region\n");
			goto error;
		}
	}

	ver = readl(iommu->reg + DMAR_VER_REG);
	pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
		drhd->reg_base_addr, DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
		iommu->cap, iommu->ecap);
	ret = iommu_init_domains(iommu);
	if (ret)
		goto error_unmap;
	spin_lock_init(&iommu->lock);
	spin_lock_init(&iommu->register_lock);

	drhd->iommu = iommu;
	return iommu;
error_unmap:
	iounmap(iommu->reg);
error:
	kfree(iommu);
	return NULL;
}

static void domain_exit(struct dmar_domain *domain);
static void free_iommu(struct intel_iommu *iommu)
{
	struct dmar_domain *domain;
	int i;

	if (!iommu)
		return;

	i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
	for (; i < cap_ndoms(iommu->cap); ) {
		domain = iommu->domains[i];
		clear_bit(i, iommu->domain_ids);
		domain_exit(domain);
		i = find_next_bit(iommu->domain_ids,
			cap_ndoms(iommu->cap), i+1);
	}

	if (iommu->gcmd & DMA_GCMD_TE)
		iommu_disable_translation(iommu);

	if (iommu->irq) {
		set_irq_data(iommu->irq, NULL);
		/* This will mask the irq */
		free_irq(iommu->irq, iommu);
		destroy_irq(iommu->irq);
	}

	kfree(iommu->domains);
	kfree(iommu->domain_ids);

	/* free context mapping */
	free_context_table(iommu);

	if (iommu->reg)
		iounmap(iommu->reg);
	kfree(iommu);
}

static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
{
	unsigned long num;
	unsigned long ndomains;
	struct dmar_domain *domain;
	unsigned long flags;

	domain = alloc_domain_mem();
	if (!domain)
		return NULL;

	ndomains = cap_ndoms(iommu->cap);

	spin_lock_irqsave(&iommu->lock, flags);
	num = find_first_zero_bit(iommu->domain_ids, ndomains);
	if (num >= ndomains) {
		spin_unlock_irqrestore(&iommu->lock, flags);
		free_domain_mem(domain);
		printk(KERN_ERR "IOMMU: no free domain ids\n");
		return NULL;
	}

	set_bit(num, iommu->domain_ids);
	domain->id = num;
	domain->iommu = iommu;
	iommu->domains[num] = domain;
	spin_unlock_irqrestore(&iommu->lock, flags);

	return domain;
}

static void iommu_free_domain(struct dmar_domain *domain)
{
	unsigned long flags;

	spin_lock_irqsave(&domain->iommu->lock, flags);
	clear_bit(domain->id, domain->iommu->domain_ids);
	spin_unlock_irqrestore(&domain->iommu->lock, flags);
}

static struct iova_domain reserved_iova_list;

static void dmar_init_reserved_ranges(void)
{
	struct pci_dev *pdev = NULL;
	struct iova *iova;
	int i;
	u64 addr, size;

	init_iova_domain(&reserved_iova_list);

	/* IOAPIC ranges shouldn't be accessed by DMA */
	iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
		IOVA_PFN(IOAPIC_RANGE_END));
	if (!iova)
		printk(KERN_ERR "Reserve IOAPIC range failed\n");

	/* Reserve all PCI MMIO to avoid peer-to-peer access */
	for_each_pci_dev(pdev) {
		struct resource *r;

		for (i = 0; i < PCI_NUM_RESOURCES; i++) {
			r = &pdev->resource[i];
			if (!r->flags || !(r->flags & IORESOURCE_MEM))
				continue;
			addr = r->start;
			addr &= PAGE_MASK_4K;
			size = r->end - addr;
			size = PAGE_ALIGN_4K(size);
			iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
				IOVA_PFN(size + addr) - 1);
			if (!iova)
				printk(KERN_ERR "Reserve iova failed\n");
		}
	}

}

static void domain_reserve_special_ranges(struct dmar_domain *domain)
{
	copy_reserved_iova(&reserved_iova_list, &domain->iovad);
}

static inline int guestwidth_to_adjustwidth(int gaw)
{
	int agaw;
	int r = (gaw - 12) % 9;

	if (r == 0)
		agaw = gaw;
	else
		agaw = gaw + 9 - r;
	if (agaw > 64)
		agaw = 64;
	return agaw;