tlbex.c

linux 内核源代码

	case CPU_VR4111:
	case CPU_VR4121:
	case CPU_VR4122:
	case CPU_VR4181:
	case CPU_VR4181A:
		i_nop(p);
		i_nop(p);
		tlbw(p);
		i_nop(p);
		i_nop(p);
		break;

	case CPU_VR4131:
	case CPU_VR4133:
	case CPU_R5432:
		i_nop(p);
		i_nop(p);
		tlbw(p);
		break;

	default:
		panic("No TLB refill handler yet (CPU type: %d)",
		      current_cpu_data.cputype);
		break;
	}
}

#ifdef CONFIG_64BIT
/*
 * TMP and PTR are scratch.
 * TMP will be clobbered, PTR will hold the pmd entry.
 */
static __init void
build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
		 unsigned int tmp, unsigned int ptr)
{
	long pgdc = (long)pgd_current;

	/*
	 * The vmalloc handling is not in the hotpath.
	 */
	i_dmfc0(p, tmp, C0_BADVADDR);
#ifdef MODULE_START
	il_bltz(p, r, tmp, label_module_alloc);
#else
	il_bltz(p, r, tmp, label_vmalloc);
#endif
	/* No i_nop needed here, since the next insn doesn't touch TMP. */

#ifdef CONFIG_SMP
# ifdef  CONFIG_MIPS_MT_SMTC
	/*
	 * SMTC uses TCBind value as "CPU" index
	 */
	i_mfc0(p, ptr, C0_TCBIND);
	i_dsrl(p, ptr, ptr, 19);
# else
	/*
	 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
	 * stored in CONTEXT.
	 */
	i_dmfc0(p, ptr, C0_CONTEXT);
	i_dsrl(p, ptr, ptr, 23);
#endif
	i_LA_mostly(p, tmp, pgdc);
	i_daddu(p, ptr, ptr, tmp);
	i_dmfc0(p, tmp, C0_BADVADDR);
	i_ld(p, ptr, rel_lo(pgdc), ptr);
#else
	i_LA_mostly(p, ptr, pgdc);
	i_ld(p, ptr, rel_lo(pgdc), ptr);
#endif

	l_vmalloc_done(l, *p);

	if (PGDIR_SHIFT - 3 < 32)		/* get pgd offset in bytes */
		i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
	else
		i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);

	i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
	i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
	i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
	i_ld(p, ptr, 0, ptr); /* get pmd pointer */
	i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
	i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
	i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
}

/*
 * BVADDR is the faulting address, PTR is scratch.
 * PTR will hold the pgd for vmalloc.
 */
static __init void
build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
			unsigned int bvaddr, unsigned int ptr)
{
	long swpd = (long)swapper_pg_dir;

#ifdef MODULE_START
	long modd = (long)module_pg_dir;

	l_module_alloc(l, *p);
	/*
	 * Assumption:
	 * VMALLOC_START >= 0xc000000000000000UL
	 * MODULE_START >= 0xe000000000000000UL
	 */
	i_SLL(p, ptr, bvaddr, 2);
	il_bgez(p, r, ptr, label_vmalloc);

	if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START)) {
		i_lui(p, ptr, rel_hi(MODULE_START)); /* delay slot */
	} else {
		/* unlikely configuration */
		i_nop(p); /* delay slot */
		i_LA(p, ptr, MODULE_START);
	}
	i_dsubu(p, bvaddr, bvaddr, ptr);

	if (in_compat_space_p(modd) && !rel_lo(modd)) {
		il_b(p, r, label_vmalloc_done);
		i_lui(p, ptr, rel_hi(modd));
	} else {
		i_LA_mostly(p, ptr, modd);
		il_b(p, r, label_vmalloc_done);
		i_daddiu(p, ptr, ptr, rel_lo(modd));
	}

	l_vmalloc(l, *p);
	if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START) &&
	    MODULE_START << 32 == VMALLOC_START)
		i_dsll32(p, ptr, ptr, 0);	/* typical case */
	else
		i_LA(p, ptr, VMALLOC_START);
#else
	l_vmalloc(l, *p);
	i_LA(p, ptr, VMALLOC_START);
#endif
	i_dsubu(p, bvaddr, bvaddr, ptr);

	if (in_compat_space_p(swpd) && !rel_lo(swpd)) {
		il_b(p, r, label_vmalloc_done);
		i_lui(p, ptr, rel_hi(swpd));
	} else {
		i_LA_mostly(p, ptr, swpd);
		il_b(p, r, label_vmalloc_done);
		i_daddiu(p, ptr, ptr, rel_lo(swpd));
	}
}

#else /* !CONFIG_64BIT */

/*
 * TMP and PTR are scratch.
 * TMP will be clobbered, PTR will hold the pgd entry.
 */
static __init void __maybe_unused
build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
{
	long pgdc = (long)pgd_current;

	/* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
#ifdef CONFIG_SMP
#ifdef  CONFIG_MIPS_MT_SMTC
	/*
	 * SMTC uses TCBind value as "CPU" index
	 */
	i_mfc0(p, ptr, C0_TCBIND);
	i_LA_mostly(p, tmp, pgdc);
	i_srl(p, ptr, ptr, 19);
#else
	/*
	 * smp_processor_id() << 3 is stored in CONTEXT.
         */
	i_mfc0(p, ptr, C0_CONTEXT);
	i_LA_mostly(p, tmp, pgdc);
	i_srl(p, ptr, ptr, 23);
#endif
	i_addu(p, ptr, tmp, ptr);
#else
	i_LA_mostly(p, ptr, pgdc);
#endif
	i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
	i_lw(p, ptr, rel_lo(pgdc), ptr);
	i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
	i_sll(p, tmp, tmp, PGD_T_LOG2);
	i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
}

#endif /* !CONFIG_64BIT */

static __init void build_adjust_context(u32 **p, unsigned int ctx)
{
	unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
	unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);

	switch (current_cpu_type()) {
	case CPU_VR41XX:
	case CPU_VR4111:
	case CPU_VR4121:
	case CPU_VR4122:
	case CPU_VR4131:
	case CPU_VR4181:
	case CPU_VR4181A:
	case CPU_VR4133:
		shift += 2;
		break;

	default:
		break;
	}

	if (shift)
		i_SRL(p, ctx, ctx, shift);
	i_andi(p, ctx, ctx, mask);
}

static __init void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
{
	/*
	 * Bug workaround for the Nevada. It seems as if under certain
	 * circumstances the move from cp0_context might produce a
	 * bogus result when the mfc0 instruction and its consumer are
	 * in a different cacheline or a load instruction, probably any
	 * memory reference, is between them.
	 */
	switch (current_cpu_type()) {
	case CPU_NEVADA:
		i_LW(p, ptr, 0, ptr);
		GET_CONTEXT(p, tmp); /* get context reg */
		break;

	default:
		GET_CONTEXT(p, tmp); /* get context reg */
		i_LW(p, ptr, 0, ptr);
		break;
	}

	build_adjust_context(p, tmp);
	i_ADDU(p, ptr, ptr, tmp); /* add in offset */
}

static __init void build_update_entries(u32 **p, unsigned int tmp,
					unsigned int ptep)
{
	/*
	 * 64bit address support (36bit on a 32bit CPU) in a 32bit
	 * Kernel is a special case. Only a few CPUs use it.
	 */
#ifdef CONFIG_64BIT_PHYS_ADDR
	if (cpu_has_64bits) {
		i_ld(p, tmp, 0, ptep); /* get even pte */
		i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
		i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
		i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
		i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
		i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
	} else {
		int pte_off_even = sizeof(pte_t) / 2;
		int pte_off_odd = pte_off_even + sizeof(pte_t);

		/* The pte entries are pre-shifted */
		i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
		i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
		i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
		i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
	}
#else
	i_LW(p, tmp, 0, ptep); /* get even pte */
	i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
	if (r45k_bvahwbug())
		build_tlb_probe_entry(p);
	i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
	if (r4k_250MHZhwbug())
		i_mtc0(p, 0, C0_ENTRYLO0);
	i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
	i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
	if (r45k_bvahwbug())
		i_mfc0(p, tmp, C0_INDEX);
	if (r4k_250MHZhwbug())
		i_mtc0(p, 0, C0_ENTRYLO1);
	i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
#endif
}

static void __init build_r4000_tlb_refill_handler(void)
{
	u32 *p = tlb_handler;
	struct label *l = labels;
	struct reloc *r = relocs;
	u32 *f;
	unsigned int final_len;
	int i;

	memset(tlb_handler, 0, sizeof(tlb_handler));
	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));
	memset(final_handler, 0, sizeof(final_handler));

	/*
	 * create the plain linear handler
	 */
	if (bcm1250_m3_war()) {
		i_MFC0(&p, K0, C0_BADVADDR);
		i_MFC0(&p, K1, C0_ENTRYHI);
		i_xor(&p, K0, K0, K1);
		i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
		il_bnez(&p, &r, K0, label_leave);
		/* No need for i_nop */
	}

#ifdef CONFIG_64BIT
	build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
#else
	build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
#endif

	build_get_ptep(&p, K0, K1);
	build_update_entries(&p, K0, K1);
	build_tlb_write_entry(&p, &l, &r, tlb_random);
	l_leave(&l, p);
	i_eret(&p); /* return from trap */

#ifdef CONFIG_64BIT
	build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
#endif

	/*
	 * Overflow check: For the 64bit handler, we need at least one
	 * free instruction slot for the wrap-around branch. In worst
	 * case, if the intended insertion point is a delay slot, we
	 * need three, with the second nop'ed and the third being
	 * unused.
	 */
	/* Loongson2 ebase is different than r4k, we have more space */
#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
	if ((p - tlb_handler) > 64)
		panic("TLB refill handler space exceeded");
#else
	if (((p - tlb_handler) > 63)
	    || (((p - tlb_handler) > 61)
		&& insn_has_bdelay(relocs, tlb_handler + 29)))
		panic("TLB refill handler space exceeded");
#endif

	/*
	 * Now fold the handler in the TLB refill handler space.
	 */
#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
	f = final_handler;
	/* Simplest case, just copy the handler. */
	copy_handler(relocs, labels, tlb_handler, p, f);
	final_len = p - tlb_handler;
#else /* CONFIG_64BIT */
	f = final_handler + 32;
	if ((p - tlb_handler) <= 32) {
		/* Just copy the handler. */
		copy_handler(relocs, labels, tlb_handler, p, f);
		final_len = p - tlb_handler;
	} else {
		u32 *split = tlb_handler + 30;

		/*
		 * Find the split point.
		 */
		if (insn_has_bdelay(relocs, split - 1))
			split--;

		/* Copy first part of the handler. */
		copy_handler(relocs, labels, tlb_handler, split, f);
		f += split - tlb_handler;

		/* Insert branch. */
		l_split(&l, final_handler);
		il_b(&f, &r, label_split);
		if (insn_has_bdelay(relocs, split))
			i_nop(&f);
		else {
			copy_handler(relocs, labels, split, split + 1, f);
			move_labels(labels, f, f + 1, -1);
			f++;
			split++;
		}

		/* Copy the rest of the handler. */
		copy_handler(relocs, labels, split, p, final_handler);
		final_len = (f - (final_handler + 32)) + (p - split);
	}
#endif /* CONFIG_64BIT */

	resolve_relocs(relocs, labels);
	pr_info("Synthesized TLB refill handler (%u instructions).\n",
		final_len);

	f = final_handler;
#if defined(CONFIG_64BIT) && !defined(CONFIG_CPU_LOONGSON2)
	if (final_len > 32)
		final_len = 64;
	else
		f = final_handler + 32;
#endif /* CONFIG_64BIT */
	pr_debug("\t.set push\n");
	pr_debug("\t.set noreorder\n");
	for (i = 0; i < final_len; i++)
		pr_debug("\t.word 0x%08x\n", f[i]);
	pr_debug("\t.set pop\n");

	memcpy((void *)ebase, final_handler, 0x100);
}

/*
 * TLB load/store/modify handlers.
 *
 * Only the fastpath gets synthesized at runtime, the slowpath for
 * do_page_fault remains normal asm.
 */
extern void tlb_do_page_fault_0(void);
extern void tlb_do_page_fault_1(void);

#define __tlb_handler_align \
	__attribute__((__aligned__(1 << CONFIG_MIPS_L1_CACHE_SHIFT)))

/*
 * 128 instructions for the fastpath handler is generous and should
 * never be exceeded.
 */
#define FASTPATH_SIZE 128

u32 __tlb_handler_align handle_tlbl[FASTPATH_SIZE];
u32 __tlb_handler_align handle_tlbs[FASTPATH_SIZE];
u32 __tlb_handler_align handle_tlbm[FASTPATH_SIZE];

static void __init
iPTE_LW(u32 **p, struct label **l, unsigned int pte, unsigned int ptr)
{
#ifdef CONFIG_SMP
# ifdef CONFIG_64BIT_PHYS_ADDR
	if (cpu_has_64bits)
		i_lld(p, pte, 0, ptr);
	else
# endif
		i_LL(p, pte, 0, ptr);
#else
# ifdef CONFIG_64BIT_PHYS_ADDR
	if (cpu_has_64bits)
		i_ld(p, pte, 0, ptr);
	else
# endif
		i_LW(p, pte, 0, ptr);
#endif
}

static void __init
iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr,
	unsigned int mode)
{
#ifdef CONFIG_64BIT_PHYS_ADDR
	unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
#endif

	i_ori(p, pte, pte, mode);
#ifdef CONFIG_SMP
# ifdef CONFIG_64BIT_PHYS_ADDR
	if (cpu_has_64bits)
		i_scd(p, pte, 0, ptr);
	else
# endif
		i_SC(p, pte, 0, ptr);

	if (r10000_llsc_war())
		il_beqzl(p, r, pte, label_smp_pgtable_change);
	else
		il_beqz(p, r, pte, label_smp_pgtable_change);

# ifdef CONFIG_64BIT_PHYS_ADDR
	if (!cpu_has_64bits) {
		/* no i_nop needed */
		i_ll(p, pte, sizeof(pte_t) / 2, ptr);
		i_ori(p, pte, pte, hwmode);