srmmu.c

内核linux2.4.20,可跟rtlinux3.2打补丁 组成实时linux系统,编译内核

/* $Id: srmmu.c,v 1.233 2001/11/13 00:49:27 davem Exp $
 * srmmu.c:  SRMMU specific routines for memory management.
 *
 * Copyright (C) 1995 David S. Miller  ([e-mail])
 * Copyright (C) 1995 Pete Zaitcev
 * Copyright (C) 1996 Eddie C. Dost    ([e-mail])
 * Copyright (C) 1997,1998 Jakub Jelinek ([e-mail])
 * Copyright (C) 1999,2000 Anton Blanchard ([e-mail])
 */

#include <linux/config.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/init.h>
#include <linux/blk.h>
#include <linux/spinlock.h>
#include <linux/bootmem.h>
#include <linux/fs.h>
#include <linux/seq_file.h>

#include <asm/page.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/io.h>
#include <asm/kdebug.h>
#include <asm/vaddrs.h>
#include <asm/traps.h>
#include <asm/smp.h>
#include <asm/mbus.h>
#include <asm/cache.h>
#include <asm/oplib.h>
#include <asm/sbus.h>
#include <asm/asi.h>
#include <asm/msi.h>
#include <asm/a.out.h>
#include <asm/mmu_context.h>
#include <asm/io-unit.h>

/* Now the cpu specific definitions. */
#include <asm/viking.h>
#include <asm/mxcc.h>
#include <asm/ross.h>
#include <asm/tsunami.h>
#include <asm/swift.h>
#include <asm/turbosparc.h>

#include <asm/btfixup.h>

enum mbus_module srmmu_modtype;
unsigned int hwbug_bitmask;
int vac_cache_size;
int vac_line_size;

extern struct resource sparc_iomap;

extern unsigned long last_valid_pfn;

extern unsigned long page_kernel;

pgd_t *srmmu_swapper_pg_dir;

#ifdef CONFIG_SMP
#define FLUSH_BEGIN(mm)
#define FLUSH_END
#else
#define FLUSH_BEGIN(mm) if((mm)->context != NO_CONTEXT) {
#define FLUSH_END	}
#endif

BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long)
#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page)

int flush_page_for_dma_global = 1;

#ifdef CONFIG_SMP
BTFIXUPDEF_CALL(void, local_flush_page_for_dma, unsigned long)
#define local_flush_page_for_dma(page) BTFIXUP_CALL(local_flush_page_for_dma)(page)
#endif

char *srmmu_name;

ctxd_t *srmmu_ctx_table_phys;
ctxd_t *srmmu_context_table;

int viking_mxcc_present;
spinlock_t srmmu_context_spinlock = SPIN_LOCK_UNLOCKED;

int is_hypersparc;

/*
 * In general all page table modifications should use the V8 atomic
 * swap instruction.  This insures the mmu and the cpu are in sync
 * with respect to ref/mod bits in the page tables.
 */
static inline unsigned long srmmu_swap(unsigned long *addr, unsigned long value)
{
	__asm__ __volatile__("swap [%2], %0" : "=&r" (value) : "0" (value), "r" (addr));
	return value;
}

static inline void srmmu_set_pte(pte_t *ptep, pte_t pteval)
{
	srmmu_swap((unsigned long *)ptep, pte_val(pteval));
}

/* The very generic SRMMU page table operations. */
static inline int srmmu_device_memory(unsigned long x)
{
	return ((x & 0xF0000000) != 0);
}

int srmmu_cache_pagetables;

/* XXX Make this dynamic based on ram size - Anton */
#define SRMMU_NOCACHE_BITMAP_SIZE (SRMMU_NOCACHE_NPAGES * 16)
#define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4)

void *srmmu_nocache_pool;
void *srmmu_nocache_bitmap;
int srmmu_nocache_low;
int srmmu_nocache_used;
spinlock_t srmmu_nocache_spinlock;

/* This makes sense. Honest it does - Anton */
#define __nocache_pa(VADDR) (((unsigned long)VADDR) - SRMMU_NOCACHE_VADDR + __pa((unsigned long)srmmu_nocache_pool))
#define __nocache_va(PADDR) (__va((unsigned long)PADDR) - (unsigned long)srmmu_nocache_pool + SRMMU_NOCACHE_VADDR)
#define __nocache_fix(VADDR) __va(__nocache_pa(VADDR))

static inline unsigned long srmmu_pgd_page(pgd_t pgd)
{ return srmmu_device_memory(pgd_val(pgd))?~0:(unsigned long)__nocache_va((pgd_val(pgd) & SRMMU_PTD_PMASK) << 4); }

static inline unsigned long srmmu_pmd_page(pmd_t pmd)
{ return srmmu_device_memory(pmd_val(pmd))?~0:(unsigned long)__nocache_va((pmd_val(pmd) & SRMMU_PTD_PMASK) << 4); }

static inline struct page *srmmu_pte_page(pte_t pte)
{ return (mem_map + (unsigned long)(srmmu_device_memory(pte_val(pte))?~0:(((pte_val(pte) & SRMMU_PTE_PMASK) << 4) >> PAGE_SHIFT))); }

static inline int srmmu_pte_none(pte_t pte)
{ return !(pte_val(pte) & 0xFFFFFFF); }

static inline int srmmu_pte_present(pte_t pte)
{ return ((pte_val(pte) & SRMMU_ET_MASK) == SRMMU_ET_PTE); }

static inline void srmmu_pte_clear(pte_t *ptep)
{ srmmu_set_pte(ptep, __pte(0)); }

static inline int srmmu_pmd_none(pmd_t pmd)
{ return !(pmd_val(pmd) & 0xFFFFFFF); }

static inline int srmmu_pmd_bad(pmd_t pmd)
{ return (pmd_val(pmd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; }

static inline int srmmu_pmd_present(pmd_t pmd)
{ return ((pmd_val(pmd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); }

static inline void srmmu_pmd_clear(pmd_t *pmdp)
{ srmmu_set_pte((pte_t *)pmdp, __pte(0)); }

static inline int srmmu_pgd_none(pgd_t pgd)          
{ return !(pgd_val(pgd) & 0xFFFFFFF); }

static inline int srmmu_pgd_bad(pgd_t pgd)
{ return (pgd_val(pgd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; }

static inline int srmmu_pgd_present(pgd_t pgd)
{ return ((pgd_val(pgd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); }

static inline void srmmu_pgd_clear(pgd_t * pgdp)
{ srmmu_set_pte((pte_t *)pgdp, __pte(0)); }

static inline int srmmu_pte_write(pte_t pte)
{ return pte_val(pte) & SRMMU_WRITE; }

static inline int srmmu_pte_dirty(pte_t pte)
{ return pte_val(pte) & SRMMU_DIRTY; }

static inline int srmmu_pte_young(pte_t pte)
{ return pte_val(pte) & SRMMU_REF; }

static inline pte_t srmmu_pte_wrprotect(pte_t pte)
{ return __pte(pte_val(pte) & ~SRMMU_WRITE);}

static inline pte_t srmmu_pte_mkclean(pte_t pte)
{ return __pte(pte_val(pte) & ~SRMMU_DIRTY);}

static inline pte_t srmmu_pte_mkold(pte_t pte)
{ return __pte(pte_val(pte) & ~SRMMU_REF);}

static inline pte_t srmmu_pte_mkwrite(pte_t pte)
{ return __pte(pte_val(pte) | SRMMU_WRITE);}

static inline pte_t srmmu_pte_mkdirty(pte_t pte)
{ return __pte(pte_val(pte) | SRMMU_DIRTY);}

static inline pte_t srmmu_pte_mkyoung(pte_t pte)
{ return __pte(pte_val(pte) | SRMMU_REF);}

/*
 * Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 */
static pte_t srmmu_mk_pte(struct page *page, pgprot_t pgprot)
{ return __pte((((page - mem_map) << PAGE_SHIFT) >> 4) | pgprot_val(pgprot)); }

static pte_t srmmu_mk_pte_phys(unsigned long page, pgprot_t pgprot)
{ return __pte(((page) >> 4) | pgprot_val(pgprot)); }

static pte_t srmmu_mk_pte_io(unsigned long page, pgprot_t pgprot, int space)
{ return __pte(((page) >> 4) | (space << 28) | pgprot_val(pgprot)); }

/* XXX should we hyper_flush_whole_icache here - Anton */
static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp)
{ srmmu_set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); }

static inline void srmmu_pgd_set(pgd_t * pgdp, pmd_t * pmdp)
{ srmmu_set_pte((pte_t *)pgdp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pmdp) >> 4))); }

static inline void srmmu_pmd_set(pmd_t * pmdp, pte_t * ptep)
{ srmmu_set_pte((pte_t *)pmdp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) ptep) >> 4))); }

static inline pte_t srmmu_pte_modify(pte_t pte, pgprot_t newprot)
{ return __pte((pte_val(pte) & SRMMU_CHG_MASK) | pgprot_val(newprot)); }

/* to find an entry in a top-level page table... */
extern inline pgd_t *srmmu_pgd_offset(struct mm_struct * mm, unsigned long address)
{ return mm->pgd + (address >> SRMMU_PGDIR_SHIFT); }

/* Find an entry in the second-level page table.. */
static inline pmd_t *srmmu_pmd_offset(pgd_t * dir, unsigned long address)
{ return (pmd_t *) srmmu_pgd_page(*dir) + ((address >> SRMMU_PMD_SHIFT) & (SRMMU_PTRS_PER_PMD - 1)); }

/* Find an entry in the third-level page table.. */ 
static inline pte_t *srmmu_pte_offset(pmd_t * dir, unsigned long address)
{ return (pte_t *) srmmu_pmd_page(*dir) + ((address >> PAGE_SHIFT) & (SRMMU_PTRS_PER_PTE - 1)); }

unsigned long __srmmu_get_nocache(int size, int align)
{
	int offset = srmmu_nocache_low;
	int i;
	unsigned long va_tmp, phys_tmp;
	int lowest_failed = 0;

	size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;

	spin_lock(&srmmu_nocache_spinlock);

repeat:
	offset = find_next_zero_bit(srmmu_nocache_bitmap, SRMMU_NOCACHE_BITMAP_SIZE, offset);

	/* we align on physical address */
	if (align) {
		va_tmp = (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT));
		phys_tmp = (__nocache_pa(va_tmp) + align - 1) & ~(align - 1);
		va_tmp = (unsigned long)__nocache_va(phys_tmp);
		offset = (va_tmp - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;
	}

	if ((SRMMU_NOCACHE_BITMAP_SIZE - offset) < size) {
		printk("Run out of nocached RAM!\n");
		spin_unlock(&srmmu_nocache_spinlock);
		return 0;
	}

	i = 0;
	while(i < size) {
		if (test_bit(offset + i, srmmu_nocache_bitmap)) {
			lowest_failed = 1;
			offset = offset + i + 1;
			goto repeat;
		}
		i++;
	}

	i = 0;
	while(i < size) {
		set_bit(offset + i, srmmu_nocache_bitmap);
		i++;
		srmmu_nocache_used++;
	}

	if (!lowest_failed && ((align >> SRMMU_NOCACHE_BITMAP_SHIFT) <= 1) && (offset > srmmu_nocache_low))
		srmmu_nocache_low = offset;

	spin_unlock(&srmmu_nocache_spinlock);

	return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT));
}

unsigned inline long srmmu_get_nocache(int size, int align)
{
	unsigned long tmp;

	tmp = __srmmu_get_nocache(size, align);

	if (tmp)
		memset((void *)tmp, 0, size);

	return tmp;
}

void srmmu_free_nocache(unsigned long vaddr, int size)
{
	int offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;

	size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;

	spin_lock(&srmmu_nocache_spinlock);

	while(size--) {
		clear_bit(offset + size, srmmu_nocache_bitmap);
		srmmu_nocache_used--;
	}

	if (offset < srmmu_nocache_low)
		srmmu_nocache_low = offset;

	spin_unlock(&srmmu_nocache_spinlock);
}

void srmmu_early_allocate_ptable_skeleton(unsigned long start, unsigned long end);

void srmmu_nocache_init(void)
{
	pgd_t *pgd;
	pmd_t *pmd;
	pte_t *pte;
	unsigned long paddr, vaddr;
	unsigned long pteval;

	srmmu_nocache_pool = __alloc_bootmem(SRMMU_NOCACHE_SIZE, PAGE_SIZE, 0UL);
	memset(srmmu_nocache_pool, 0, SRMMU_NOCACHE_SIZE);

	srmmu_nocache_bitmap = __alloc_bootmem(SRMMU_NOCACHE_BITMAP_SIZE, SMP_CACHE_BYTES, 0UL);
	memset(srmmu_nocache_bitmap, 0, SRMMU_NOCACHE_BITMAP_SIZE);

	srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
	memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
	init_mm.pgd = srmmu_swapper_pg_dir;

	srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, SRMMU_NOCACHE_END);

	spin_lock_init(&srmmu_nocache_spinlock);

	paddr = __pa((unsigned long)srmmu_nocache_pool);
	vaddr = SRMMU_NOCACHE_VADDR;

	while (vaddr < SRMMU_NOCACHE_END) {
		pgd = pgd_offset_k(vaddr);
		pmd = srmmu_pmd_offset(__nocache_fix(pgd), vaddr);
		pte = srmmu_pte_offset(__nocache_fix(pmd), vaddr);

		pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV);

		if (srmmu_cache_pagetables)
			pteval |= SRMMU_CACHE;

		srmmu_set_pte(__nocache_fix(pte), pteval);

		vaddr += PAGE_SIZE;
		paddr += PAGE_SIZE;
	}

	flush_cache_all();
	flush_tlb_all();
}

static inline pgd_t *srmmu_get_pgd_fast(void)
{
	pgd_t *pgd = NULL;

	pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
	if (pgd) {
		pgd_t *init = pgd_offset_k(0);
		memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
		memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
						(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
	}

	return pgd;
}

static void srmmu_free_pgd_fast(pgd_t *pgd)
{
	srmmu_free_nocache((unsigned long)pgd, SRMMU_PGD_TABLE_SIZE);
}

static pte_t *srmmu_pte_alloc_one_fast(struct mm_struct *mm, unsigned long address)
{
	return (pte_t *)srmmu_get_nocache(SRMMU_PTE_TABLE_SIZE, SRMMU_PTE_TABLE_SIZE);
}

static pte_t *srmmu_pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
	BUG();
	return NULL;
}

static void srmmu_free_pte_fast(pte_t *pte)
{
	srmmu_free_nocache((unsigned long)pte, SRMMU_PTE_TABLE_SIZE);
}

static pmd_t *srmmu_pmd_alloc_one_fast(struct mm_struct *mm, unsigned long address)
{
	return (pmd_t *)srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
}

static void srmmu_free_pmd_fast(pmd_t * pmd)
{
	srmmu_free_nocache((unsigned long)pmd, SRMMU_PMD_TABLE_SIZE);
}

static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
{
	struct ctx_list *ctxp;

	ctxp = ctx_free.next;
	if(ctxp != &ctx_free) {
		remove_from_ctx_list(ctxp);
		add_to_used_ctxlist(ctxp);
		mm->context = ctxp->ctx_number;