mm.c

xen 3.2.2 源码

            if ( !_shadow_mode_refcounts(domain) && !domain->is_dying )
                gdprintk(XENLOG_INFO,
                         "Error pfn %lx: rd=%p, od=%p, caf=%08x, taf=%"
                         PRtype_info "\n",
                         page_to_mfn(page), domain, unpickle_domptr(d),
                         x, page->u.inuse.type_info);
            return 0;
        }
        asm volatile (
            LOCK_PREFIX "cmpxchg8b %3"
            : "=d" (nd), "=a" (y), "=c" (d),
            "=m" (*(volatile u64 *)(&page->count_info))
            : "0" (d), "1" (x), "c" (d), "b" (nx) );
    }
    while ( unlikely(nd != d) || unlikely(y != x) );

    return 1;
}


static int alloc_page_type(struct page_info *page, unsigned long type)
{
    struct domain *owner = page_get_owner(page);

    /* A page table is dirtied when its type count becomes non-zero. */
    if ( likely(owner != NULL) )
        paging_mark_dirty(owner, page_to_mfn(page));

    switch ( type & PGT_type_mask )
    {
    case PGT_l1_page_table:
        return alloc_l1_table(page);
    case PGT_l2_page_table:
        return alloc_l2_table(page, type);
    case PGT_l3_page_table:
        return alloc_l3_table(page);
    case PGT_l4_page_table:
        return alloc_l4_table(page);
    case PGT_gdt_page:
    case PGT_ldt_page:
        return alloc_segdesc_page(page);
    default:
        printk("Bad type in alloc_page_type %lx t=%" PRtype_info " c=%x\n", 
               type, page->u.inuse.type_info,
               page->count_info);
        BUG();
    }

    return 0;
}


void free_page_type(struct page_info *page, unsigned long type)
{
    struct domain *owner = page_get_owner(page);
    unsigned long gmfn;

    if ( likely(owner != NULL) )
    {
        /*
         * We have to flush before the next use of the linear mapping
         * (e.g., update_va_mapping()) or we could end up modifying a page
         * that is no longer a page table (and hence screw up ref counts).
         */
        if ( current->domain == owner )
            queue_deferred_ops(owner, DOP_FLUSH_ALL_TLBS);
        else
            flush_tlb_mask(owner->domain_dirty_cpumask);

        if ( unlikely(paging_mode_enabled(owner)) )
        {
            /* A page table is dirtied when its type count becomes zero. */
            paging_mark_dirty(owner, page_to_mfn(page));

            if ( shadow_mode_refcounts(owner) )
                return;

            gmfn = mfn_to_gmfn(owner, page_to_mfn(page));
            ASSERT(VALID_M2P(gmfn));
            shadow_remove_all_shadows(owner->vcpu[0], _mfn(gmfn));
        }
    }

    switch ( type & PGT_type_mask )
    {
    case PGT_l1_page_table:
        free_l1_table(page);
        break;

    case PGT_l2_page_table:
        free_l2_table(page);
        break;

#if CONFIG_PAGING_LEVELS >= 3
    case PGT_l3_page_table:
        free_l3_table(page);
        break;
#endif

#if CONFIG_PAGING_LEVELS >= 4
    case PGT_l4_page_table:
        free_l4_table(page);
        break;
#endif

    default:
        printk("%s: type %lx pfn %lx\n",__FUNCTION__,
               type, page_to_mfn(page));
        BUG();
    }
}


void put_page_type(struct page_info *page)
{
    unsigned long nx, x, y = page->u.inuse.type_info;

 again:
    do {
        x  = y;
        nx = x - 1;

        ASSERT((x & PGT_count_mask) != 0);

        if ( unlikely((nx & PGT_count_mask) == 0) )
        {
            if ( unlikely((nx & PGT_type_mask) <= PGT_l4_page_table) &&
                 likely(nx & PGT_validated) )
            {
                /*
                 * Page-table pages must be unvalidated when count is zero. The
                 * 'free' is safe because the refcnt is non-zero and validated
                 * bit is clear => other ops will spin or fail.
                 */
                if ( unlikely((y = cmpxchg(&page->u.inuse.type_info, x, 
                                           x & ~PGT_validated)) != x) )
                    goto again;
                /* We cleared the 'valid bit' so we do the clean up. */
                free_page_type(page, x);
                /* Carry on, but with the 'valid bit' now clear. */
                x  &= ~PGT_validated;
                nx &= ~PGT_validated;
            }

            /*
             * Record TLB information for flush later. We do not stamp page
             * tables when running in shadow mode:
             *  1. Pointless, since it's the shadow pt's which must be tracked.
             *  2. Shadow mode reuses this field for shadowed page tables to
             *     store flags info -- we don't want to conflict with that.
             */
            if ( !(shadow_mode_enabled(page_get_owner(page)) &&
                   (page->count_info & PGC_page_table)) )
                page->tlbflush_timestamp = tlbflush_current_time();
        }
    }
    while ( unlikely((y = cmpxchg(&page->u.inuse.type_info, x, nx)) != x) );
}


int get_page_type(struct page_info *page, unsigned long type)
{
    unsigned long nx, x, y = page->u.inuse.type_info;

    ASSERT(!(type & ~(PGT_type_mask | PGT_pae_xen_l2)));

 again:
    do {
        x  = y;
        nx = x + 1;
        if ( unlikely((nx & PGT_count_mask) == 0) )
        {
            MEM_LOG("Type count overflow on pfn %lx", page_to_mfn(page));
            return 0;
        }
        else if ( unlikely((x & PGT_count_mask) == 0) )
        {
            struct domain *d = page_get_owner(page);

            /* Never allow a shadowed frame to go from type count 0 to 1 */
            if ( d && shadow_mode_enabled(d) )
                shadow_remove_all_shadows(d->vcpu[0], _mfn(page_to_mfn(page)));

            ASSERT(!(x & PGT_pae_xen_l2));
            if ( (x & PGT_type_mask) != type )
            {
                /*
                 * On type change we check to flush stale TLB entries. This 
                 * may be unnecessary (e.g., page was GDT/LDT) but those 
                 * circumstances should be very rare.
                 */
                cpumask_t mask = d->domain_dirty_cpumask;

                /* Don't flush if the timestamp is old enough */
                tlbflush_filter(mask, page->tlbflush_timestamp);

                if ( unlikely(!cpus_empty(mask)) &&
                     /* Shadow mode: track only writable pages. */
                     (!shadow_mode_enabled(page_get_owner(page)) ||
                      ((nx & PGT_type_mask) == PGT_writable_page)) )
                {
                    perfc_incr(need_flush_tlb_flush);
                    flush_tlb_mask(mask);
                }

                /* We lose existing type and validity. */
                nx &= ~(PGT_type_mask | PGT_validated);
                nx |= type;

                /* No special validation needed for writable pages. */
                /* Page tables and GDT/LDT need to be scanned for validity. */
                if ( type == PGT_writable_page )
                    nx |= PGT_validated;
            }
        }
        else if ( unlikely((x & (PGT_type_mask|PGT_pae_xen_l2)) != type) )
        {
            /* Don't log failure if it could be a recursive-mapping attempt. */
            if ( ((x & PGT_type_mask) == PGT_l2_page_table) &&
                 (type == PGT_l1_page_table) )
                return 0;
            if ( ((x & PGT_type_mask) == PGT_l3_page_table) &&
                 (type == PGT_l2_page_table) )
                return 0;
            if ( ((x & PGT_type_mask) == PGT_l4_page_table) &&
                 (type == PGT_l3_page_table) )
                return 0;
            MEM_LOG("Bad type (saw %" PRtype_info " != exp %" PRtype_info ") "
                    "for mfn %lx (pfn %lx)",
                    x, type, page_to_mfn(page),
                    get_gpfn_from_mfn(page_to_mfn(page)));
            return 0;
        }
        else if ( unlikely(!(x & PGT_validated)) )
        {
            /* Someone else is updating validation of this page. Wait... */
            while ( (y = page->u.inuse.type_info) == x )
                cpu_relax();
            goto again;
        }
    }
    while ( unlikely((y = cmpxchg(&page->u.inuse.type_info, x, nx)) != x) );

    if ( unlikely(!(nx & PGT_validated)) )
    {
        /* Try to validate page type; drop the new reference on failure. */
        if ( unlikely(!alloc_page_type(page, type)) )
        {
            MEM_LOG("Error while validating mfn %lx (pfn %lx) for type %"
                    PRtype_info ": caf=%08x taf=%" PRtype_info,
                    page_to_mfn(page), get_gpfn_from_mfn(page_to_mfn(page)),
                    type, page->count_info, page->u.inuse.type_info);
            /* Noone else can get a reference. We hold the only ref. */
            page->u.inuse.type_info = 0;
            return 0;
        }

        /* Noone else is updating simultaneously. */
        __set_bit(_PGT_validated, &page->u.inuse.type_info);
    }

    return 1;
}


void cleanup_page_cacheattr(struct page_info *page)
{
    uint32_t cacheattr = (page->count_info >> PGC_cacheattr_base) & 7;

    if ( likely(cacheattr == 0) )
        return;

    page->count_info &= ~PGC_cacheattr_mask;

    BUG_ON(is_xen_heap_page(page));

#ifdef __x86_64__
    map_pages_to_xen((unsigned long)page_to_virt(page), page_to_mfn(page),
                     1, PAGE_HYPERVISOR);
#endif
}


int new_guest_cr3(unsigned long mfn)
{
    struct vcpu *v = current;
    struct domain *d = v->domain;
    int okay;
    unsigned long old_base_mfn;

#ifdef CONFIG_COMPAT
    if ( is_pv_32on64_domain(d) )
    {
        okay = paging_mode_refcounts(d)
            ? 0 /* Old code was broken, but what should it be? */
            : mod_l4_entry(
                    __va(pagetable_get_paddr(v->arch.guest_table)),
                    l4e_from_pfn(
                        mfn,
                        (_PAGE_PRESENT|_PAGE_RW|_PAGE_USER|_PAGE_ACCESSED)),
                    pagetable_get_pfn(v->arch.guest_table));
        if ( unlikely(!okay) )
        {
            MEM_LOG("Error while installing new compat baseptr %lx", mfn);
            return 0;
        }

        invalidate_shadow_ldt(v);
        write_ptbase(v);

        return 1;
    }
#endif
    okay = paging_mode_refcounts(d)
        ? get_page_from_pagenr(mfn, d)
        : get_page_and_type_from_pagenr(mfn, PGT_root_page_table, d);
    if ( unlikely(!okay) )
    {
        MEM_LOG("Error while installing new baseptr %lx", mfn);
        return 0;
    }

    invalidate_shadow_ldt(v);

    old_base_mfn = pagetable_get_pfn(v->arch.guest_table);

    v->arch.guest_table = pagetable_from_pfn(mfn);
    update_cr3(v);

    write_ptbase(v);

    if ( likely(old_base_mfn != 0) )
    {
        if ( paging_mode_refcounts(d) )
            put_page(mfn_to_page(old_base_mfn));
        else
            put_page_and_type(mfn_to_page(old_base_mfn));
    }

    return 1;
}

static void process_deferred_ops(void)
{
    unsigned int deferred_ops;
    struct domain *d = current->domain;
    struct percpu_mm_info *info = &this_cpu(percpu_mm_info);

    deferred_ops = info->deferred_ops;
    info->deferred_ops = 0;

    if ( deferred_ops & (DOP_FLUSH_ALL_TLBS|DOP_FLUSH_TLB) )
    {
        if ( deferred_ops & DOP_FLUSH_ALL_TLBS )
            flush_tlb_mask(d->domain_dirty_cpumask);
        else
            flush_tlb_local();
    }

    if ( deferred_ops & DOP_RELOAD_LDT )
        (void)map_ldt_shadow_page(0);

    if ( unlikely(info->foreign != NULL) )
    {
        rcu_unlock_domain(info->foreign);
        info->foreign = NULL;
    }
}

static int set_foreigndom(domid_t domid)
{
    struct domain *e, *d = current->domain;
    struct percpu_mm_info *info = &this_cpu(percpu_mm_info);
    int okay = 1;

    ASSERT(info->foreign == NULL);

    if ( likely(domid == DOMID_SELF) )
        goto out;

    if ( unlikely(domid == d->domain_id) )
    {
        MEM_LOG("Cannot specify itself as foreign domain");
        okay = 0;
    }
    else if ( unlikely(paging_mode_translate(d)) )
    {
        MEM_LOG("Cannot mix foreign mappings with translated domains");
        okay = 0;
    }
    else if ( !IS_PRIV(d) )
    {
        switch ( domid )
        {
        case DOMID_IO:
            info->foreign = rcu_lock_domain(dom_io);
            break;
        default:
            MEM_LOG("Cannot set foreign dom");
            okay = 0;
            break;
        }
    }
    else
    {
        info->foreign = e = rcu_lock_domain_by_id(domid);
        if ( e == NULL )
        {
            switch ( domid )
            {
            case DOMID_XEN:
                info->foreign = rcu_lock_domain(