c-r4k.c

linux-2.4.29操作系统的源码

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1996 David S. Miller ([e-mail])
 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle ([e-mail])
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 */
#include <linux/config.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/bitops.h>

#include <asm/bcache.h>
#include <asm/bootinfo.h>
#include <asm/cacheops.h>
#include <asm/cpu.h>
#include <asm/io.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/r4kcache.h>
#include <asm/system.h>
#include <asm/mmu_context.h>
#include <asm/war.h>

static unsigned long icache_size, dcache_size, scache_size;

/*
 * Dummy cache handling routines for machines without boardcaches
 */
static void no_sc_noop(void) {}

static struct bcache_ops no_sc_ops = {
	.bc_enable = (void *)no_sc_noop,
	.bc_disable = (void *)no_sc_noop,
	.bc_wback_inv = (void *)no_sc_noop,
	.bc_inv = (void *)no_sc_noop
};

struct bcache_ops *bcops = &no_sc_ops;

#define cpu_is_r4600_v1_x()	((read_c0_prid() & 0xfffffff0) == 0x2010)
#define cpu_is_r4600_v2_x()	((read_c0_prid() & 0xfffffff0) == 0x2020)

#define R4600_HIT_CACHEOP_WAR_IMPL					\
do {									\
	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())		\
		*(volatile unsigned long *)KSEG1;			\
	if (R4600_V1_HIT_CACHEOP_WAR)					\
		__asm__ __volatile__("nop;nop;nop;nop");		\
} while (0)

static void (* r4k_blast_dcache_page)(unsigned long addr);

static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
{
	R4600_HIT_CACHEOP_WAR_IMPL;
	blast_dcache32_page(addr);
}

static inline void r4k_blast_dcache_page_setup(void)
{
	unsigned long dc_lsize = current_cpu_data.dcache.linesz;

	if (dc_lsize == 16)
		r4k_blast_dcache_page = blast_dcache16_page;
	else if (dc_lsize == 32)
		r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
}

static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);

static void r4k_blast_dcache_page_indexed_setup(void)
{
	unsigned long dc_lsize = current_cpu_data.dcache.linesz;

	if (dc_lsize == 16)
		r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
	else if (dc_lsize == 32)
		r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
}

static void (* r4k_blast_dcache)(void);

static inline void r4k_blast_dcache_setup(void)
{
	unsigned long dc_lsize = current_cpu_data.dcache.linesz;

	if (dc_lsize == 16)
		r4k_blast_dcache = blast_dcache16;
	else if (dc_lsize == 32)
		r4k_blast_dcache = blast_dcache32;
}

/* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
#define JUMP_TO_ALIGN(order) \
	__asm__ __volatile__( \
		"b\t1f\n\t" \
		".align\t" #order "\n\t" \
		"1:\n\t" \
		)
#define CACHE32_UNROLL32_ALIGN	JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
#define CACHE32_UNROLL32_ALIGN2	JUMP_TO_ALIGN(11)

static inline void blast_r4600_v1_icache32(void)
{
	unsigned long flags;

	local_irq_save(flags);
	blast_icache32();
	local_irq_restore(flags);
}

static inline void tx49_blast_icache32(void)
{
	unsigned long start = KSEG0;
	unsigned long end = start + current_cpu_data.icache.waysize;
	unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
	unsigned long ws_end = current_cpu_data.icache.ways <<
	                       current_cpu_data.icache.waybit;
	unsigned long ws, addr;

	CACHE32_UNROLL32_ALIGN2;
	/* I'm in even chunk.  blast odd chunks */
	for (ws = 0; ws < ws_end; ws += ws_inc) 
		for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 
			cache32_unroll32(addr|ws,Index_Invalidate_I);
	CACHE32_UNROLL32_ALIGN;
	/* I'm in odd chunk.  blast even chunks */
	for (ws = 0; ws < ws_end; ws += ws_inc) 
		for (addr = start; addr < end; addr += 0x400 * 2) 
			cache32_unroll32(addr|ws,Index_Invalidate_I);
}

static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
{
	unsigned long flags;

	local_irq_save(flags);
	blast_icache32_page_indexed(page);
	local_irq_restore(flags);
}

static inline void tx49_blast_icache32_page_indexed(unsigned long page)
{
	unsigned long start = page;
	unsigned long end = start + PAGE_SIZE;
	unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
	unsigned long ws_end = current_cpu_data.icache.ways <<
	                       current_cpu_data.icache.waybit;
	unsigned long ws, addr;

	CACHE32_UNROLL32_ALIGN2;
	/* I'm in even chunk.  blast odd chunks */
	for (ws = 0; ws < ws_end; ws += ws_inc) 
		for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 
			cache32_unroll32(addr|ws,Index_Invalidate_I);
	CACHE32_UNROLL32_ALIGN;
	/* I'm in odd chunk.  blast even chunks */
	for (ws = 0; ws < ws_end; ws += ws_inc) 
		for (addr = start; addr < end; addr += 0x400 * 2) 
			cache32_unroll32(addr|ws,Index_Invalidate_I);
}

static void (* r4k_blast_icache_page)(unsigned long addr);

static inline void r4k_blast_icache_page_setup(void)
{
	unsigned long ic_lsize = current_cpu_data.icache.linesz;

	if (ic_lsize == 16)
		r4k_blast_icache_page = blast_icache16_page;
	else if (ic_lsize == 32)
		r4k_blast_icache_page = blast_icache32_page;
	else if (ic_lsize == 64)
		r4k_blast_icache_page = blast_icache64_page;
}

static void (* r4k_blast_icache_page_indexed)(unsigned long addr);

static inline void r4k_blast_icache_page_indexed_setup(void)
{
	unsigned long ic_lsize = current_cpu_data.icache.linesz;

	if (ic_lsize == 16)
		r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
	else if (ic_lsize == 32) {
		if (TX49XX_ICACHE_INDEX_INV_WAR)
			r4k_blast_icache_page_indexed =
				tx49_blast_icache32_page_indexed;
		else if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
			r4k_blast_icache_page_indexed =
				blast_icache32_r4600_v1_page_indexed;
		else
			r4k_blast_icache_page_indexed =
				blast_icache32_page_indexed;
	} else if (ic_lsize == 64)
		r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
}

static void (* r4k_blast_icache)(void);

static inline void r4k_blast_icache_setup(void)
{
	unsigned long ic_lsize = current_cpu_data.icache.linesz;

	if (ic_lsize == 16)
		r4k_blast_icache = blast_icache16;
	else if (ic_lsize == 32) {
		if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
			r4k_blast_icache = blast_r4600_v1_icache32;
		else if (TX49XX_ICACHE_INDEX_INV_WAR)
			r4k_blast_icache = tx49_blast_icache32;
		else if (ic_lsize == 32)
			r4k_blast_icache = blast_icache32;
	} else if (ic_lsize == 64)
		r4k_blast_icache = blast_icache64;
}

static void (* r4k_blast_scache_page)(unsigned long addr);

static inline void r4k_blast_scache_page_setup(void)
{
	unsigned long sc_lsize = current_cpu_data.scache.linesz;

	if (sc_lsize == 16)
		r4k_blast_scache_page = blast_scache16_page;
	else if (sc_lsize == 32)
		r4k_blast_scache_page = blast_scache32_page;
	else if (sc_lsize == 64)
		r4k_blast_scache_page = blast_scache64_page;
	else if (sc_lsize == 128)
		r4k_blast_scache_page = blast_scache128_page;
}

static void (* r4k_blast_scache)(void);

static inline void r4k_blast_scache_setup(void)
{
	unsigned long sc_lsize = current_cpu_data.scache.linesz;

	if (sc_lsize == 16)
		r4k_blast_scache = blast_scache16;
	else if (sc_lsize == 32)
		r4k_blast_scache = blast_scache32;
	else if (sc_lsize == 64)
		r4k_blast_scache = blast_scache64;
	else if (sc_lsize == 128)
		r4k_blast_scache = blast_scache128;
}

static void r4k_flush_cache_all(void)
{
	if (!cpu_has_dc_aliases)
		return;

	r4k_blast_dcache();
	r4k_blast_icache();
}

static void r4k___flush_cache_all(void)
{
	r4k_blast_dcache();
	r4k_blast_icache();

	switch (current_cpu_data.cputype) {
	case CPU_R4000SC:
	case CPU_R4000MC:
	case CPU_R4400SC:
	case CPU_R4400MC:
	case CPU_R10000:
	case CPU_R12000:
		r4k_blast_scache();
	}
}

static void r4k_flush_cache_range(struct mm_struct *mm,
	unsigned long start, unsigned long end)
{
	if (!cpu_has_dc_aliases)
		return;

	if (cpu_context(smp_processor_id(), mm) != 0) {
		r4k_blast_dcache();
		r4k_blast_icache();
	}
}

static void r4k_flush_cache_mm(struct mm_struct *mm)
{
	if (!cpu_has_dc_aliases)
		return;

	if (!cpu_context(smp_processor_id(), mm))
		return;

	r4k_blast_dcache();
	r4k_blast_icache();

	/*
	 * Kludge alert.  For obscure reasons R4000SC and R4400SC go nuts if we
	 * only flush the primary caches but R10000 and R12000 behave sane ...
	 */
	if (current_cpu_data.cputype == CPU_R4000SC ||
	    current_cpu_data.cputype == CPU_R4000MC ||
	    current_cpu_data.cputype == CPU_R4400SC ||
	    current_cpu_data.cputype == CPU_R4400MC)
		r4k_blast_scache();
}

static void r4k_flush_cache_page(struct vm_area_struct *vma,
					unsigned long page)
{
	int exec = vma->vm_flags & VM_EXEC;
	struct mm_struct *mm = vma->vm_mm;
	pgd_t *pgdp;
	pmd_t *pmdp;
	pte_t *ptep;

	/*
	 * If ownes no valid ASID yet, cannot possibly have gotten
	 * this page into the cache.
	 */
	if (cpu_context(smp_processor_id(), mm) == 0)
		return;

	page &= PAGE_MASK;
	pgdp = pgd_offset(mm, page);
	pmdp = pmd_offset(pgdp, page);
	ptep = pte_offset(pmdp, page);

	/*
	 * If the page isn't marked valid, the page cannot possibly be
	 * in the cache.
	 */
	if (!(pte_val(*ptep) & _PAGE_PRESENT))
		return;

	/*
	 * Doing flushes for another ASID than the current one is
	 * too difficult since stupid R4k caches do a TLB translation
	 * for every cache flush operation.  So we do indexed flushes
	 * in that case, which doesn't overly flush the cache too much.
	 */
	if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
		if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
			r4k_blast_dcache_page(page);
		if (exec)
			r4k_blast_icache_page(page);

		return;
	}

	/*
	 * Do indexed flush, too much work to get the (possible) TLB refills
	 * to work correctly.
	 */
	page = (KSEG0 + (page & (dcache_size - 1)));
	if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
		r4k_blast_dcache_page_indexed(page);
	if (exec) {
		if (cpu_has_vtag_icache) {
			int cpu = smp_processor_id();

			if (cpu_context(cpu, vma->vm_mm) != 0)
				drop_mmu_context(vma->vm_mm, cpu);
		} else
			r4k_blast_icache_page_indexed(page);
	}
}

static void r4k_flush_data_cache_page(unsigned long addr)
{
	r4k_blast_dcache_page(addr);
}

static void r4k_flush_icache_range(unsigned long start, unsigned long end)
{
	unsigned long dc_lsize = current_cpu_data.dcache.linesz;
	unsigned long ic_lsize = current_cpu_data.icache.linesz;
	unsigned long addr, aend;

	if (!cpu_has_ic_fills_f_dc) {
		if (end - start > dcache_size)
			r4k_blast_dcache();
		else {
			addr = start & ~(dc_lsize - 1);
			aend = (end - 1) & ~(dc_lsize - 1);

			while (1) {
				/* Hit_Writeback_Inv_D */
				protected_writeback_dcache_line(addr);
				if (addr == aend)
					break;
				addr += dc_lsize;
			}
		}
	}

	if (end - start > icache_size)
		r4k_blast_icache();
	else {
		addr = start & ~(ic_lsize - 1);
		aend = (end - 1) & ~(ic_lsize - 1);
		while (1) {
			/* Hit_Invalidate_I */
			protected_flush_icache_line(addr);
			if (addr == aend)
				break;
			addr += ic_lsize;
		}
	}
}

/*
 * Ok, this seriously sucks.  We use them to flush a user page but don't
 * know the virtual address, so we have to blast away the whole icache
 * which is significantly more expensive than the real thing.  Otoh we at
 * least know the kernel address of the page so we can flush it
 * selectivly.
 */
static void r4k_flush_icache_page(struct vm_area_struct *vma,
	struct page *page)
{
	/*
	 * If there's no context yet, or the page isn't executable, no icache
	 * flush is needed.
	 */
	if (!(vma->vm_flags & VM_EXEC))
		return;

	/*
	 * Tricky ...  Because we don't know the virtual address we've got the
	 * choice of either invalidating the entire primary and secondary
	 * caches or invalidating the secondary caches also.  With the subset
	 * enforcment on R4000SC, R4400SC, R10000 and R12000 invalidating the
	 * secondary cache will result in any entries in the primary caches
	 * also getting invalidated which hopefully is a bit more economical.
	 */
	if (cpu_has_subset_pcaches) {
		unsigned long addr = (unsigned long) page_address(page);

		r4k_blast_scache_page(addr);
		ClearPageDcacheDirty(page);

		return;
	}

	if (!cpu_has_ic_fills_f_dc) {
		unsigned long addr = (unsigned long) page_address(page);
		r4k_blast_dcache_page(addr);
		ClearPageDcacheDirty(page);
	}

	/*
	 * We're not sure of the virtual address(es) involved here, so
	 * we have to flush the entire I-cache.
	 */
	if (cpu_has_vtag_icache) {
		int cpu = smp_processor_id();

		if (cpu_context(cpu, vma->vm_mm) != 0)
			drop_mmu_context(vma->vm_mm, cpu);
	} else
		r4k_blast_icache();
}

#ifdef CONFIG_NONCOHERENT_IO

static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
{
	unsigned long end, a;

	/* Catch bad driver code */
	BUG_ON(size == 0);

	if (cpu_has_subset_pcaches) {
		unsigned long sc_lsize = current_cpu_data.scache.linesz;

		if (size >= scache_size) {
			r4k_blast_scache();
			return;
		}

		a = addr & ~(sc_lsize - 1);
		end = (addr + size - 1) & ~(sc_lsize - 1);
		while (1) {
			flush_scache_line(a);	/* Hit_Writeback_Inv_SD */
			if (a == end)
				break;
			a += sc_lsize;
		}
		return;
	}

	/*
	 * Either no secondary cache or the available caches don't have the
	 * subset property so we have to flush the primary caches
	 * explicitly
	 */
	if (size >= dcache_size) {
		r4k_blast_dcache();
	} else {
		unsigned long dc_lsize = current_cpu_data.dcache.linesz;

		R4600_HIT_CACHEOP_WAR_IMPL;
		a = addr & ~(dc_lsize - 1);
		end = (addr + size - 1) & ~(dc_lsize - 1);
		while (1) {
			flush_dcache_line(a);	/* Hit_Writeback_Inv_D */
			if (a == end)
				break;
			a += dc_lsize;
		}
	}

	bc_wback_inv(addr, size);
}

static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
{
	unsigned long end, a;

	/* Catch bad driver code */
	BUG_ON(size == 0);

	if (cpu_has_subset_pcaches) {
		unsigned long sc_lsize = current_cpu_data.scache.linesz;

		if (size >= scache_size) {
			r4k_blast_scache();
			return;
		}

		a = addr & ~(sc_lsize - 1);
		end = (addr + size - 1) & ~(sc_lsize - 1);
		while (1) {
			flush_scache_line(a);	/* Hit_Writeback_Inv_SD */
			if (a == end)
				break;
			a += sc_lsize;
		}
		return;
	}

	if (size >= dcache_size) {
		r4k_blast_dcache();
	} else {
		unsigned long dc_lsize = current_cpu_data.dcache.linesz;

		R4600_HIT_CACHEOP_WAR_IMPL;
		a = addr & ~(dc_lsize - 1);
		end = (addr + size - 1) & ~(dc_lsize - 1);
		while (1) {
			flush_dcache_line(a);	/* Hit_Writeback_Inv_D */
			if (a == end)