tlbex.c

linux-2.6.15.6

	move_relocs(rel, first, end, off);
	move_labels(lab, first, end, off);
}

static __init int __attribute__((unused)) insn_has_bdelay(struct reloc *rel,
							  u32 *addr)
{
	for (; rel->lab != label_invalid; rel++) {
		if (rel->addr == addr
		    && (rel->type == R_MIPS_PC16
			|| rel->type == R_MIPS_26))
			return 1;
	}

	return 0;
}

/* convenience functions for labeled branches */
static void __attribute__((unused)) il_bltz(u32 **p, struct reloc **r,
					    unsigned int reg, enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_bltz(p, reg, 0);
}

static void __attribute__((unused)) il_b(u32 **p, struct reloc **r,
					 enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_b(p, 0);
}

static void il_beqz(u32 **p, struct reloc **r, unsigned int reg,
		    enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_beqz(p, reg, 0);
}

static void __attribute__((unused))
il_beqzl(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_beqzl(p, reg, 0);
}

static void il_bnez(u32 **p, struct reloc **r, unsigned int reg,
		    enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_bnez(p, reg, 0);
}

static void il_bgezl(u32 **p, struct reloc **r, unsigned int reg,
		     enum label_id l)
{
	r_mips_pc16(r, *p, l);
	i_bgezl(p, reg, 0);
}

/* The only general purpose registers allowed in TLB handlers. */
#define K0		26
#define K1		27

/* Some CP0 registers */
#define C0_INDEX	0
#define C0_ENTRYLO0	2
#define C0_ENTRYLO1	3
#define C0_CONTEXT	4
#define C0_BADVADDR	8
#define C0_ENTRYHI	10
#define C0_EPC		14
#define C0_XCONTEXT	20

#ifdef CONFIG_64BIT
# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_XCONTEXT)
#else
# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_CONTEXT)
#endif

/* The worst case length of the handler is around 18 instructions for
 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
 * Maximum space available is 32 instructions for R3000 and 64
 * instructions for R4000.
 *
 * We deliberately chose a buffer size of 128, so we won't scribble
 * over anything important on overflow before we panic.
 */
static __initdata u32 tlb_handler[128];

/* simply assume worst case size for labels and relocs */
static __initdata struct label labels[128];
static __initdata struct reloc relocs[128];

/*
 * The R3000 TLB handler is simple.
 */
static void __init build_r3000_tlb_refill_handler(void)
{
	long pgdc = (long)pgd_current;
	u32 *p;

	memset(tlb_handler, 0, sizeof(tlb_handler));
	p = tlb_handler;

	i_mfc0(&p, K0, C0_BADVADDR);
	i_lui(&p, K1, rel_hi(pgdc)); /* cp0 delay */
	i_lw(&p, K1, rel_lo(pgdc), K1);
	i_srl(&p, K0, K0, 22); /* load delay */
	i_sll(&p, K0, K0, 2);
	i_addu(&p, K1, K1, K0);
	i_mfc0(&p, K0, C0_CONTEXT);
	i_lw(&p, K1, 0, K1); /* cp0 delay */
	i_andi(&p, K0, K0, 0xffc); /* load delay */
	i_addu(&p, K1, K1, K0);
	i_lw(&p, K0, 0, K1);
	i_nop(&p); /* load delay */
	i_mtc0(&p, K0, C0_ENTRYLO0);
	i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
	i_tlbwr(&p); /* cp0 delay */
	i_jr(&p, K1);
	i_rfe(&p); /* branch delay */

	if (p > tlb_handler + 32)
		panic("TLB refill handler space exceeded");

	printk("Synthesized TLB refill handler (%u instructions).\n",
	       (unsigned int)(p - tlb_handler));
#ifdef DEBUG_TLB
	{
		int i;

		for (i = 0; i < (p - tlb_handler); i++)
			printk("%08x\n", tlb_handler[i]);
	}
#endif

	memcpy((void *)CAC_BASE, tlb_handler, 0x80);
}

/*
 * The R4000 TLB handler is much more complicated. We have two
 * consecutive handler areas with 32 instructions space each.
 * Since they aren't used at the same time, we can overflow in the
 * other one.To keep things simple, we first assume linear space,
 * then we relocate it to the final handler layout as needed.
 */
static __initdata u32 final_handler[64];

/*
 * Hazards
 *
 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
 * 2. A timing hazard exists for the TLBP instruction.
 *
 *      stalling_instruction
 *      TLBP
 *
 * The JTLB is being read for the TLBP throughout the stall generated by the
 * previous instruction. This is not really correct as the stalling instruction
 * can modify the address used to access the JTLB.  The failure symptom is that
 * the TLBP instruction will use an address created for the stalling instruction
 * and not the address held in C0_ENHI and thus report the wrong results.
 *
 * The software work-around is to not allow the instruction preceding the TLBP
 * to stall - make it an NOP or some other instruction guaranteed not to stall.
 *
 * Errata 2 will not be fixed.  This errata is also on the R5000.
 *
 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
 */
static __init void __attribute__((unused)) build_tlb_probe_entry(u32 **p)
{
	switch (current_cpu_data.cputype) {
	/* Found by experiment: R4600 v2.0 needs this, too.  */
	case CPU_R4600:
	case CPU_R5000:
	case CPU_R5000A:
	case CPU_NEVADA:
		i_nop(p);
		i_tlbp(p);
		break;

	default:
		i_tlbp(p);
		break;
	}
}

/*
 * Write random or indexed TLB entry, and care about the hazards from
 * the preceeding mtc0 and for the following eret.
 */
enum tlb_write_entry { tlb_random, tlb_indexed };

static __init void build_tlb_write_entry(u32 **p, struct label **l,
					 struct reloc **r,
					 enum tlb_write_entry wmode)
{
	void(*tlbw)(u32 **) = NULL;

	switch (wmode) {
	case tlb_random: tlbw = i_tlbwr; break;
	case tlb_indexed: tlbw = i_tlbwi; break;
	}

	switch (current_cpu_data.cputype) {
	case CPU_R4000PC:
	case CPU_R4000SC:
	case CPU_R4000MC:
	case CPU_R4400PC:
	case CPU_R4400SC:
	case CPU_R4400MC:
		/*
		 * This branch uses up a mtc0 hazard nop slot and saves
		 * two nops after the tlbw instruction.
		 */
		il_bgezl(p, r, 0, label_tlbw_hazard);
		tlbw(p);
		l_tlbw_hazard(l, *p);
		i_nop(p);
		break;

	case CPU_R4600:
	case CPU_R4700:
	case CPU_R5000:
	case CPU_R5000A:
		i_nop(p);
		tlbw(p);
		i_nop(p);
		break;

	case CPU_R4300:
	case CPU_5KC:
	case CPU_TX49XX:
	case CPU_AU1000:
	case CPU_AU1100:
	case CPU_AU1500:
	case CPU_AU1550:
	case CPU_AU1200:
	case CPU_PR4450:
		i_nop(p);
		tlbw(p);
		break;

	case CPU_R10000:
	case CPU_R12000:
	case CPU_4KC:
	case CPU_SB1:
	case CPU_SB1A:
	case CPU_4KSC:
	case CPU_20KC:
	case CPU_25KF:
		tlbw(p);
		break;

	case CPU_NEVADA:
		i_nop(p); /* QED specifies 2 nops hazard */
		/*
		 * This branch uses up a mtc0 hazard nop slot and saves
		 * a nop after the tlbw instruction.
		 */
		il_bgezl(p, r, 0, label_tlbw_hazard);
		tlbw(p);
		l_tlbw_hazard(l, *p);
		break;

	case CPU_RM7000:
		i_nop(p);
		i_nop(p);
		i_nop(p);
		i_nop(p);
		tlbw(p);
		break;

	case CPU_4KEC:
	case CPU_24K:
	case CPU_34K:
		i_ehb(p);
		tlbw(p);
		break;

	case CPU_RM9000:
		/*
		 * When the JTLB is updated by tlbwi or tlbwr, a subsequent
		 * use of the JTLB for instructions should not occur for 4
		 * cpu cycles and use for data translations should not occur
		 * for 3 cpu cycles.
		 */
		i_ssnop(p);
		i_ssnop(p);
		i_ssnop(p);
		i_ssnop(p);
		tlbw(p);
		i_ssnop(p);
		i_ssnop(p);
		i_ssnop(p);
		i_ssnop(p);
		break;

	case CPU_VR4111:
	case CPU_VR4121:
	case CPU_VR4122:
	case CPU_VR4181:
	case CPU_VR4181A:
		i_nop(p);
		i_nop(p);
		tlbw(p);
		i_nop(p);
		i_nop(p);
		break;

	case CPU_VR4131:
	case CPU_VR4133:
	case CPU_R5432:
		i_nop(p);
		i_nop(p);
		tlbw(p);
		break;

	default:
		panic("No TLB refill handler yet (CPU type: %d)",
		      current_cpu_data.cputype);
		break;
	}
}

#ifdef CONFIG_64BIT
/*
 * TMP and PTR are scratch.
 * TMP will be clobbered, PTR will hold the pmd entry.
 */
static __init void
build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
		 unsigned int tmp, unsigned int ptr)
{
	long pgdc = (long)pgd_current;

	/*
	 * The vmalloc handling is not in the hotpath.
	 */
	i_dmfc0(p, tmp, C0_BADVADDR);
	il_bltz(p, r, tmp, label_vmalloc);
	/* No i_nop needed here, since the next insn doesn't touch TMP. */

#ifdef CONFIG_SMP
# ifdef CONFIG_BUILD_ELF64
	/*
	 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
	 * stored in CONTEXT.
	 */
	i_dmfc0(p, ptr, C0_CONTEXT);
	i_dsrl(p, ptr, ptr, 23);
	i_LA_mostly(p, tmp, pgdc);
	i_daddu(p, ptr, ptr, tmp);
	i_dmfc0(p, tmp, C0_BADVADDR);
	i_ld(p, ptr, rel_lo(pgdc), ptr);
# else
	/*
	 * 64 bit SMP running in compat space has the lower part of
	 * &pgd_current[smp_processor_id()] stored in CONTEXT.
	 */
	if (!in_compat_space_p(pgdc))
		panic("Invalid page directory address!");

	i_dmfc0(p, ptr, C0_CONTEXT);
	i_dsra(p, ptr, ptr, 23);
	i_ld(p, ptr, 0, ptr);
# endif
#else
	i_LA_mostly(p, ptr, pgdc);
	i_ld(p, ptr, rel_lo(pgdc), ptr);
#endif

	l_vmalloc_done(l, *p);
	i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3); /* get pgd offset in bytes */
	i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
	i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
	i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
	i_ld(p, ptr, 0, ptr); /* get pmd pointer */
	i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
	i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
	i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
}

/*
 * BVADDR is the faulting address, PTR is scratch.
 * PTR will hold the pgd for vmalloc.
 */
static __init void
build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
			unsigned int bvaddr, unsigned int ptr)
{
	long swpd = (long)swapper_pg_dir;

	l_vmalloc(l, *p);
	i_LA(p, ptr, VMALLOC_START);
	i_dsubu(p, bvaddr, bvaddr, ptr);

	if (in_compat_space_p(swpd) && !rel_lo(swpd)) {
		il_b(p, r, label_vmalloc_done);
		i_lui(p, ptr, rel_hi(swpd));
	} else {
		i_LA_mostly(p, ptr, swpd);
		il_b(p, r, label_vmalloc_done);
		i_daddiu(p, ptr, ptr, rel_lo(swpd));
	}
}

#else /* !CONFIG_64BIT */

/*
 * TMP and PTR are scratch.
 * TMP will be clobbered, PTR will hold the pgd entry.
 */
static __init void __attribute__((unused))
build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
{
	long pgdc = (long)pgd_current;

	/* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
#ifdef CONFIG_SMP
	i_mfc0(p, ptr, C0_CONTEXT);
	i_LA_mostly(p, tmp, pgdc);
	i_srl(p, ptr, ptr, 23);
	i_addu(p, ptr, tmp, ptr);
#else
	i_LA_mostly(p, ptr, pgdc);
#endif
	i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
	i_lw(p, ptr, rel_lo(pgdc), ptr);
	i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
	i_sll(p, tmp, tmp, PGD_T_LOG2);
	i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
}

#endif /* !CONFIG_64BIT */

static __init void build_adjust_context(u32 **p, unsigned int ctx)
{
	unsigned int shift = 4 - (PTE_T_LOG2 + 1);
	unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);

	switch (current_cpu_data.cputype) {
	case CPU_VR41XX:
	case CPU_VR4111:
	case CPU_VR4121:
	case CPU_VR4122:
	case CPU_VR4131:
	case CPU_VR4181:
	case CPU_VR4181A:
	case CPU_VR4133:
		shift += 2;
		break;

	default:
		break;
	}

	if (shift)
		i_SRL(p, ctx, ctx, shift);
	i_andi(p, ctx, ctx, mask);
}

static __init void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
{
	/*
	 * Bug workaround for the Nevada. It seems as if under certain
	 * circumstances the move from cp0_context might produce a
	 * bogus result when the mfc0 instruction and its consumer are
	 * in a different cacheline or a load instruction, probably any
	 * memory reference, is between them.
	 */
	switch (current_cpu_data.cputype) {
	case CPU_NEVADA:
		i_LW(p, ptr, 0, ptr);
		GET_CONTEXT(p, tmp); /* get context reg */
		break;

	default:
		GET_CONTEXT(p, tmp); /* get context reg */
		i_LW(p, ptr, 0, ptr);
		break;
	}

	build_adjust_context(p, tmp);
	i_ADDU(p, ptr, ptr, tmp); /* add in offset */
}

static __init void build_update_entries(u32 **p, unsigned int tmp,
					unsigned int ptep)
{
	/*
	 * 64bit address support (36bit on a 32bit CPU) in a 32bit
	 * Kernel is a special case. Only a few CPUs use it.
	 */
#ifdef CONFIG_64BIT_PHYS_ADDR
	if (cpu_has_64bits) {
		i_ld(p, tmp, 0, ptep); /* get even pte */
		i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
		i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
		i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
		i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
		i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
	} else {
		int pte_off_even = sizeof(pte_t) / 2;
		int pte_off_odd = pte_off_even + sizeof(pte_t);

		/* The pte entries are pre-shifted */
		i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
		i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
		i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
		i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
	}
#else
	i_LW(p, tmp, 0, ptep); /* get even pte */
	i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
	if (r45k_bvahwbug())
		build_tlb_probe_entry(p);
	i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
	if (r4k_250MHZhwbug())
		i_mtc0(p, 0, C0_ENTRYLO0);
	i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
	i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
	if (r45k_bvahwbug())
		i_mfc0(p, tmp, C0_INDEX);
	if (r4k_250MHZhwbug())
		i_mtc0(p, 0, C0_ENTRYLO1);
	i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
#endif
}

static void __init build_r4000_tlb_refill_handler(void)
{
	u32 *p = tlb_handler;
	struct label *l = labels;
	struct reloc *r = relocs;
	u32 *f;
	unsigned int final_len;

	memset(tlb_handler, 0, sizeof(tlb_handler));
	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));
	memset(final_handler, 0, sizeof(final_handler));

	/*
	 * create the plain linear handler
	 */
	if (bcm1250_m3_war()) {
		i_MFC0(&p, K0, C0_BADVADDR);
		i_MFC0(&p, K1, C0_ENTRYHI);
		i_xor(&p, K0, K0, K1);
		i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
		il_bnez(&p, &r, K0, label_leave);
		/* No need for i_nop */
	}

#ifdef CONFIG_64BIT
	build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
#else
	build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
#endif

	build_get_ptep(&p, K0, K1);
	build_update_entries(&p, K0, K1);
	build_tlb_write_entry(&p, &l, &r, tlb_random);
	l_leave(&l, p);
	i_eret(&p); /* return from trap */

#ifdef CONFIG_64BIT
	build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
#endif

	/*
	 * Overflow check: For the 64bit handler, we need at least one
	 * free instruction slot for the wrap-around branch. In worst
	 * case, if the intended insertion point is a delay slot, we
	 * need three, with the the second nop'ed and the third being
	 * unused.
	 */
#ifdef CONFIG_32BIT
	if ((p - tlb_handler) > 64)
		panic("TLB refill handler space exceeded");
#else
	if (((p - tlb_handler) > 63)
	    || (((p - tlb_handler) > 61)
		&& insn_has_bdelay(relocs, tlb_handler + 29)))
		panic("TLB refill handler space exceeded");
#endif

	/*
	 * Now fold the handler in the TLB refill handler space.
	 */
#ifdef CONFIG_32BIT
	f = final_handler;
	/* Simplest case, just copy the handler. */
	copy_handler(relocs, labels, tlb_handler, p, f);
	final_len = p - tlb_handler;
#else /* CONFIG_64BIT */
	f = final_handler + 32;
	if ((p - tlb_handler) <= 32) {
		/* Just copy the handler. */
		copy_handler(relocs, labels, tlb_handler, p, f);
		final_len = p - tlb_handler;
	} else {