intel-iommu.c

xen 3.2.2 源码

        val |= DMA_TLB_WRITE_DRAIN;

    spin_lock_irqsave(&iommu->register_lock, flag);
    /* Note: Only uses first TLB reg currently */
    if ( val_iva )
        dmar_writeq(iommu->reg, tlb_offset, val_iva);
    dmar_writeq(iommu->reg, tlb_offset + 8, val);

    /* Make sure hardware complete it */
    start_time = jiffies;
    for ( ; ; )
    {
        val = dmar_readq(iommu->reg, tlb_offset + 8);
        if ( !(val & DMA_TLB_IVT) )
            break;
        if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
            panic("DMAR hardware is malfunctional, please disable IOMMU\n");
        cpu_relax();
    }
    spin_unlock_irqrestore(&iommu->register_lock, flag);

    /* check IOTLB invalidation granularity */
    if ( DMA_TLB_IAIG(val) == 0 )
        printk(KERN_ERR VTDPREFIX "IOMMU: flush IOTLB failed\n");
    if ( DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type) )
        printk(KERN_ERR VTDPREFIX "IOMMU: tlb flush request %x, actual %x\n",
               (u32)DMA_TLB_IIRG(type), (u32)DMA_TLB_IAIG(val));
    /* flush context entry will implictly flush write buffer */
    return 0;
}

static int inline iommu_flush_iotlb_global(struct iommu *iommu,
                                           int non_present_entry_flush)
{
    return __iommu_flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
                               non_present_entry_flush);
}

static int inline iommu_flush_iotlb_dsi(struct iommu *iommu, u16 did,
                                        int non_present_entry_flush)
{
    return __iommu_flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
                               non_present_entry_flush);
}

static int inline get_alignment(u64 base, unsigned int size)
{
    int t = 0;
    u64 end;

    end = base + size - 1;
    while ( base != end )
    {
        t++;
        base >>= 1;
        end >>= 1;
    }
    return t;
}

static int inline iommu_flush_iotlb_psi(
    struct iommu *iommu, u16 did,
    u64 addr, unsigned int pages, int non_present_entry_flush)
{
    unsigned int align;

    BUG_ON(addr & (~PAGE_MASK_4K));
    BUG_ON(pages == 0);

    /* Fallback to domain selective flush if no PSI support */
    if ( !cap_pgsel_inv(iommu->cap) )
        return iommu_flush_iotlb_dsi(iommu, did,
                                     non_present_entry_flush);

    /*
     * PSI requires page size is 2 ^ x, and the base address is naturally
     * aligned to the size
     */
    align = get_alignment(addr >> PAGE_SHIFT_4K, pages);
    /* Fallback to domain selective flush if size is too big */
    if ( align > cap_max_amask_val(iommu->cap) )
        return iommu_flush_iotlb_dsi(iommu, did,
                                     non_present_entry_flush);

    addr >>= PAGE_SHIFT_4K + align;
    addr <<= PAGE_SHIFT_4K + align;

    return __iommu_flush_iotlb(iommu, did, addr, align,
                               DMA_TLB_PSI_FLUSH, non_present_entry_flush);
}

void iommu_flush_all(void)
{
    struct acpi_drhd_unit *drhd;
    struct iommu *iommu;

    wbinvd();
    for_each_drhd_unit ( drhd )
    {
        iommu = drhd->iommu;
        iommu_flush_context_global(iommu, 0);
        iommu_flush_iotlb_global(iommu, 0);
    }
}

/* clear one page's page table */
static void dma_pte_clear_one(struct domain *domain, u64 addr)
{
    struct hvm_iommu *hd = domain_hvm_iommu(domain);
    struct acpi_drhd_unit *drhd;
    struct iommu *iommu;
    struct dma_pte *pte = NULL;
    struct page_info *pg = NULL;

    /* get last level pte */
    pg = dma_addr_level_page(domain, addr, 1);
    if ( !pg )
        return;
    pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
    pte += address_level_offset(addr, 1);
    if ( pte )
    {
        dma_clear_pte(*pte);
        iommu_flush_cache_entry(pte);

        for_each_drhd_unit ( drhd )
        {
            iommu = drhd->iommu;

            if ( !test_bit(iommu->index, &hd->iommu_bitmap) )
                continue;

            if ( cap_caching_mode(iommu->cap) )
                iommu_flush_iotlb_psi(iommu, domain_iommu_domid(domain),
                                      addr, 1, 0);
            else if (cap_rwbf(iommu->cap))
                iommu_flush_write_buffer(iommu);
        }
    }
    unmap_domain_page(pte);
}

/* clear last level pte, a tlb flush should be followed */
static void dma_pte_clear_range(struct domain *domain, u64 start, u64 end)
{
    struct hvm_iommu *hd = domain_hvm_iommu(domain);
    int addr_width = agaw_to_width(hd->agaw);

    start &= (((u64)1) << addr_width) - 1;
    end &= (((u64)1) << addr_width) - 1;
    /* in case it's partial page */
    start = PAGE_ALIGN_4K(start);
    end &= PAGE_MASK_4K;

    /* we don't need lock here, nobody else touches the iova range */
    while ( start < end )
    {
        dma_pte_clear_one(domain, start);
        start += PAGE_SIZE_4K;
    }
}

/* free page table pages. last level pte should already be cleared */
void dma_pte_free_pagetable(struct domain *domain, u64 start, u64 end)
{
    struct hvm_iommu *hd = domain_hvm_iommu(domain);
    int addr_width = agaw_to_width(hd->agaw);
    struct dma_pte *pte;
    int total = agaw_to_level(hd->agaw);
    int level;
    u64 tmp;
    struct page_info *pg = NULL;

    start &= (((u64)1) << addr_width) - 1;
    end &= (((u64)1) << addr_width) - 1;

    /* we don't need lock here, nobody else touches the iova range */
    level = 2;
    while ( level <= total )
    {
        tmp = align_to_level(start, level);
        if ( (tmp >= end) || ((tmp + level_size(level)) > end) )
            return;

        while ( tmp < end )
        {
            pg = dma_addr_level_page(domain, tmp, level);
            if ( !pg )
            {
                tmp += level_size(level);
                continue;
            }
            pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
            pte += address_level_offset(tmp, level);
            dma_clear_pte(*pte);
            iommu_flush_cache_entry(pte);
            unmap_domain_page(pte);
            free_domheap_page(pg);

            tmp += level_size(level);
        }
        level++;
    }

    /* free pgd */
    if ( start == 0 && end == ((((u64)1) << addr_width) - 1) )
    {
        free_xenheap_page((void *)hd->pgd);
        hd->pgd = NULL;
    }
}

/* iommu handling */
static int iommu_set_root_entry(struct iommu *iommu)
{
    void *addr;
    u32 cmd, sts;
    struct root_entry *root;
    unsigned long flags;
    unsigned long start_time;

    if ( iommu == NULL )
    {
        gdprintk(XENLOG_ERR VTDPREFIX,
                 "iommu_set_root_entry: iommu == NULL\n");
        return -EINVAL;
    }

    if ( unlikely(!iommu->root_entry) )
    {
        root = (struct root_entry *)alloc_xenheap_page();
        if ( root == NULL )
            return -ENOMEM;

        memset((u8*)root, 0, PAGE_SIZE);
        iommu_flush_cache_page(root);

        if ( cmpxchg((unsigned long *)&iommu->root_entry,
                     0, (unsigned long)root) != 0 )
            free_xenheap_page((void *)root);
    }

    addr = iommu->root_entry;

    spin_lock_irqsave(&iommu->register_lock, flags);

    dmar_writeq(iommu->reg, DMAR_RTADDR_REG, virt_to_maddr(addr));
    cmd = iommu->gcmd | DMA_GCMD_SRTP;
    dmar_writel(iommu->reg, DMAR_GCMD_REG, cmd);

    /* Make sure hardware complete it */
    start_time = jiffies;
    for ( ; ; )
    {
        sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
        if ( sts & DMA_GSTS_RTPS )
            break;
        if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
            panic("DMAR hardware is malfunctional, please disable IOMMU\n");
        cpu_relax();
    }

    spin_unlock_irqrestore(&iommu->register_lock, flags);

    return 0;
}

static int iommu_enable_translation(struct iommu *iommu)
{
    u32 sts;
    unsigned long flags;
    unsigned long start_time;

    dprintk(XENLOG_INFO VTDPREFIX,
            "iommu_enable_translation: enabling vt-d translation\n");
    spin_lock_irqsave(&iommu->register_lock, flags);
    iommu->gcmd |= DMA_GCMD_TE;
    dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
    /* Make sure hardware complete it */
    start_time = jiffies;
    for ( ; ; )
    {
        sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
        if ( sts & DMA_GSTS_TES )
            break;
        if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
            panic("DMAR hardware is malfunctional, please disable IOMMU\n");
        cpu_relax();
    }

    /* Disable PMRs when VT-d engine takes effect per spec definition */
    disable_pmr(iommu);
    spin_unlock_irqrestore(&iommu->register_lock, flags);
    return 0;
}

int iommu_disable_translation(struct iommu *iommu)
{
    u32 sts;
    unsigned long flags;
    unsigned long start_time;

    spin_lock_irqsave(&iommu->register_lock, flags);
    iommu->gcmd &= ~ DMA_GCMD_TE;
    dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);

    /* Make sure hardware complete it */
    start_time = jiffies;
    for ( ; ; )
    {
        sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
        if ( !(sts & DMA_GSTS_TES) )
            break;
        if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
            panic("DMAR hardware is malfunctional, please disable IOMMU\n");
        cpu_relax();
    }
    spin_unlock_irqrestore(&iommu->register_lock, flags);
    return 0;
}

static struct iommu *vector_to_iommu[NR_VECTORS];
static int iommu_page_fault_do_one(struct iommu *iommu, int type,
                                   u8 fault_reason, u16 source_id, u32 addr)
{
    dprintk(XENLOG_WARNING VTDPREFIX,
            "iommu_page_fault:%s: DEVICE %x:%x.%x addr %x REASON %x\n",
            (type ? "DMA Read" : "DMA Write"),
            (source_id >> 8), PCI_SLOT(source_id & 0xFF),
            PCI_FUNC(source_id & 0xFF), addr, fault_reason);

 /* For 32-bit Xen, maddr_to_virt() can not be used in print_vtd_entries()
  * since the pages (except the pgd) are allocated from domheap.
  */
#ifndef __i386__
    print_vtd_entries(current->domain, (source_id >> 8),(source_id & 0xff),
                      (addr >> PAGE_SHIFT)); 
#endif
    return 0;
}

#define PRIMARY_FAULT_REG_LEN (16)
static void iommu_page_fault(int vector, void *dev_id,
                             struct cpu_user_regs *regs)
{
    struct iommu *iommu = dev_id;
    int reg, fault_index;
    u32 fault_status;
    unsigned long flags;

    dprintk(XENLOG_WARNING VTDPREFIX,
            "iommu_page_fault: iommu->reg = %p\n", iommu->reg);

    spin_lock_irqsave(&iommu->register_lock, flags);
    fault_status = dmar_readl(iommu->reg, DMAR_FSTS_REG);
    spin_unlock_irqrestore(&iommu->register_lock, flags);

    /* FIXME: ignore advanced fault log */
    if ( !(fault_status & DMA_FSTS_PPF) )
        return;
    fault_index = dma_fsts_fault_record_index(fault_status);
    reg = cap_fault_reg_offset(iommu->cap);
    for ( ; ; )
    {
        u8 fault_reason;
        u16 source_id;
        u32 guest_addr, data;
        int type;

        /* highest 32 bits */
        spin_lock_irqsave(&iommu->register_lock, flags);
        data = dmar_readl(iommu->reg, reg +
                          fault_index * PRIMARY_FAULT_REG_LEN + 12);
        if ( !(data & DMA_FRCD_F) )
        {
            spin_unlock_irqrestore(&iommu->register_lock, flags);
            break;
        }

        fault_reason = dma_frcd_fault_reason(data);
        type = dma_frcd_type(data);

        data = dmar_readl(iommu->reg, reg +
                          fault_index * PRIMARY_FAULT_REG_LEN + 8);
        source_id = dma_frcd_source_id(data);

        guest_addr = dmar_readq(iommu->reg, reg +
                                fault_index * PRIMARY_FAULT_REG_LEN);
        guest_addr = dma_frcd_page_addr(guest_addr);
        /* clear the fault */
        dmar_writel(iommu->reg, reg +
                    fault_index * PRIMARY_FAULT_REG_LEN + 12, DMA_FRCD_F);
        spin_unlock_irqrestore(&iommu->register_lock, flags);

        iommu_page_fault_do_one(iommu, type, fault_reason,
                                source_id, guest_addr);

        fault_index++;
        if ( fault_index > cap_num_fault_regs(iommu->cap) )
            fault_index = 0;
    }

    /* clear primary fault overflow */
    if ( fault_status & DMA_FSTS_PFO )
    {
        spin_lock_irqsave(&iommu->register_lock, flags);
        dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
        spin_unlock_irqrestore(&iommu->register_lock, flags);
    }
}

static void dma_msi_unmask(unsigned int vector)
{
    struct iommu *iommu = vector_to_iommu[vector];
    unsigned long flags;

    /* unmask it */
    spin_lock_irqsave(&iommu->register_lock, flags);
    dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
    spin_unlock_irqrestore(&iommu->register_lock, flags);
}

static void dma_msi_mask(unsigned int vector)
{
    unsigned long flags;
    struct iommu *iommu = vector_to_iommu[vector];

    /* mask it */
    spin_lock_irqsave(&iommu->register_lock, flags);
    dmar_writel(iommu->reg, DMAR_FECTL_REG, DMA_FECTL_IM);
    spin_unlock_irqrestore(&iommu->register_lock, flags);
}

static unsigned int dma_msi_startup(unsigned int vector)
{
    dma_msi_unmask(vector);