blockops.s

该文件是rt_linux

/* $Id: blockops.S,v 1.41 2001/12/05 06:05:35 davem Exp $
 * blockops.S: UltraSparc block zero optimized routines.
 *
 * Copyright (C) 1996, 1998, 1999, 2000 David S. Miller ([e-mail])
 * Copyright (C) 1997 Jakub Jelinek ([e-mail])
 */

#include "VIS.h"
#include <asm/visasm.h>
#include <asm/page.h>
#include <asm/dcu.h>
#include <asm/spitfire.h>
#include <asm/pgtable.h>
#include <asm/asm_offsets.h>

#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7)	\
	fmovd	%reg0, %f48; 	fmovd	%reg1, %f50;		\
	fmovd	%reg2, %f52; 	fmovd	%reg3, %f54;		\
	fmovd	%reg4, %f56; 	fmovd	%reg5, %f58;		\
	fmovd	%reg6, %f60; 	fmovd	%reg7, %f62;

#define	DCACHE_SIZE	(PAGE_SIZE * 2)
#define TLBTEMP_ENT1	(60 << 3)
#define TLBTEMP_ENT2	(61 << 3)
#define TLBTEMP_ENTSZ	(1 << 3)

#if (PAGE_SHIFT == 13) || (PAGE_SHIFT == 19)
#define PAGE_SIZE_REM	0x80
#elif (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
#define PAGE_SIZE_REM	0x100
#else
#error Wrong PAGE_SHIFT specified
#endif

	.text

	.align		32
	.globl		copy_user_page
	.type		copy_user_page,@function
copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
	VISEntry
	sethi		%hi(PAGE_SIZE), %g3
	sub		%o0, %g4, %g1
	and		%o2, %g3, %o0
	sethi		%hi(TLBTEMP_BASE), %o3
	sethi		%uhi(_PAGE_VALID | _PAGE_SZBITS), %g3
	sub		%o1, %g4, %g2
	sllx		%g3, 32, %g3
	mov		TLB_TAG_ACCESS, %o2
	or		%g3, (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), %g3
	sethi		%hi(DCACHE_SIZE), %o1
	or		%g1, %g3, %g1
	or		%g2, %g3, %g2
	add		%o0, %o3, %o0
	add		%o0, %o1, %o1
#define FIX_INSN_1	0x96102060 /* mov (12 << 3), %o3 */
cheetah_patch_1:
	mov		TLBTEMP_ENT1, %o3
	rdpr		%pstate, %g3
	wrpr		%g3, PSTATE_IE, %pstate

	/* Do this now, before loading the fixed TLB entries for copying,
	 * so we do not risk a multiple TLB match condition later when
	 * restoring those entries.
	 */
	ldub		[%g6 + AOFF_task_thread + AOFF_thread_use_blkcommit], %g3

	/* Spitfire Errata #32 workaround */
	mov		PRIMARY_CONTEXT, %o4
	stxa		%g0, [%o4] ASI_DMMU
	membar		#Sync

	ldxa		[%o3] ASI_DTLB_TAG_READ, %o4

	/* Spitfire Errata #32 workaround */
	mov		PRIMARY_CONTEXT, %o5
	stxa		%g0, [%o5] ASI_DMMU
	membar		#Sync

	ldxa		[%o3] ASI_DTLB_DATA_ACCESS, %g0
	ldxa		[%o3] ASI_DTLB_DATA_ACCESS, %o5
	stxa		%o0, [%o2] ASI_DMMU
	stxa		%g1, [%o3] ASI_DTLB_DATA_ACCESS
	membar		#Sync
	add		%o3, (TLBTEMP_ENTSZ), %o3

	/* Spitfire Errata #32 workaround */
	mov		PRIMARY_CONTEXT, %g5
	stxa		%g0, [%g5] ASI_DMMU
	membar		#Sync

	ldxa		[%o3] ASI_DTLB_TAG_READ, %g5

	/* Spitfire Errata #32 workaround */
	mov		PRIMARY_CONTEXT, %g7
	stxa		%g0, [%g7] ASI_DMMU
	membar		#Sync

	ldxa		[%o3] ASI_DTLB_DATA_ACCESS, %g0
	ldxa		[%o3] ASI_DTLB_DATA_ACCESS, %g7
	stxa		%o1, [%o2] ASI_DMMU
	stxa		%g2, [%o3] ASI_DTLB_DATA_ACCESS
	membar		#Sync

	cmp		%g3, 0
	bne,pn		%xcc, copy_page_using_blkcommit
	 nop

	BRANCH_IF_ANY_CHEETAH(g3,o2,cheetah_copy_user_page)
	ba,pt		%xcc, spitfire_copy_user_page
	 nop

cheetah_copy_user_page:
	.globl		cheetah_copy_user_page_nop_1_6
cheetah_copy_user_page_nop_1_6:
	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g3
	sethi		%uhi(DCU_PE), %o2
	sllx		%o2, 32, %o2
	or		%g3, %o2, %o2
	stxa		%o2, [%g0] ASI_DCU_CONTROL_REG	! Enable P-cache
	membar		#Sync

	sethi		%hi((PAGE_SIZE/64)-7), %o2	! A0 Group
	prefetch	[%o1 + 0x000], #one_read	! MS
	or		%o2, %lo((PAGE_SIZE/64)-7), %o2	! A1 Group
	prefetch	[%o1 + 0x040], #one_read	! MS
	prefetch	[%o1 + 0x080], #one_read	! MS Group
	prefetch	[%o1 + 0x0c0], #one_read	! MS Group
	ldd		[%o1 + 0x000], %f0		! MS Group
	prefetch	[%o1 + 0x100], #one_read	! MS Group
	ldd		[%o1 + 0x008], %f2		! AX
	prefetch	[%o1 + 0x140], #one_read	! MS Group
	ldd		[%o1 + 0x010], %f4		! AX
	prefetch	[%o1 + 0x180], #one_read	! MS Group
	fmovd		%f0, %f32			! FGA Group
	ldd		[%o1 + 0x018], %f6		! AX
	fmovd		%f2, %f34			! FGA Group
	ldd		[%o1 + 0x020], %f8		! MS
	fmovd		%f4, %f36			! FGA Group
	ldd		[%o1 + 0x028], %f10		! AX
	membar		#StoreStore			! MS
	fmovd		%f6, %f38			! FGA Group
	ldd		[%o1 + 0x030], %f12		! MS
	fmovd		%f8, %f40			! FGA Group
	ldd		[%o1 + 0x038], %f14		! AX
	fmovd		%f10, %f42			! FGA Group
	ldd		[%o1 + 0x040], %f16		! MS
1:	ldd		[%o1 + 0x048], %f2		! AX (Group)
	fmovd		%f12, %f44			! FGA
	ldd		[%o1 + 0x050], %f4		! MS
	fmovd		%f14, %f46			! FGA Group
	stda		%f32, [%o0] ASI_BLK_P		! MS
	ldd		[%o1 + 0x058], %f6		! AX
	fmovd		%f16, %f32			! FGA Group (8-cycle stall)
	ldd		[%o1 + 0x060], %f8		! MS
	fmovd		%f2, %f34			! FGA Group
	ldd		[%o1 + 0x068], %f10		! AX
	fmovd		%f4, %f36			! FGA Group
	ldd		[%o1 + 0x070], %f12		! MS
	fmovd		%f6, %f38			! FGA Group
	ldd		[%o1 + 0x078], %f14		! AX
	fmovd		%f8, %f40			! FGA Group
	ldd		[%o1 + 0x080], %f16		! AX
	prefetch	[%o1 + 0x180], #one_read	! MS
	fmovd		%f10, %f42			! FGA Group
	subcc		%o2, 1, %o2			! A0
	add		%o0, 0x40, %o0			! A1
	bne,pt		%xcc, 1b			! BR
	 add		%o1, 0x40, %o1			! A0 Group

	mov		5, %o2				! A0 Group
1:	ldd		[%o1 + 0x048], %f2		! AX
	fmovd		%f12, %f44			! FGA
	ldd		[%o1 + 0x050], %f4		! MS
	fmovd		%f14, %f46			! FGA Group
	stda		%f32, [%o0] ASI_BLK_P		! MS
	ldd		[%o1 + 0x058], %f6		! AX
	fmovd		%f16, %f32			! FGA Group (8-cycle stall)
	ldd		[%o1 + 0x060], %f8		! MS
	fmovd		%f2, %f34			! FGA Group
	ldd		[%o1 + 0x068], %f10		! AX
	fmovd		%f4, %f36			! FGA Group
	ldd		[%o1 + 0x070], %f12		! MS
	fmovd		%f6, %f38			! FGA Group
	ldd		[%o1 + 0x078], %f14		! AX
	fmovd		%f8, %f40			! FGA Group
	ldd		[%o1 + 0x080], %f16		! MS
	fmovd		%f10, %f42			! FGA Group
	subcc		%o2, 1, %o2			! A0
	add		%o0, 0x40, %o0			! A1
	bne,pt		%xcc, 1b			! BR
	 add		%o1, 0x40, %o1			! A0 Group

	ldd		[%o1 + 0x048], %f2		! AX
	fmovd		%f12, %f44			! FGA
	ldd		[%o1 + 0x050], %f4		! MS
	fmovd		%f14, %f46			! FGA Group
	stda		%f32, [%o0] ASI_BLK_P		! MS
	ldd		[%o1 + 0x058], %f6		! AX
	fmovd		%f16, %f32			! FGA Group (8-cycle stall)
	ldd		[%o1 + 0x060], %f8		! MS
	fmovd		%f2, %f34			! FGA Group
	ldd		[%o1 + 0x068], %f10		! AX
	fmovd		%f4, %f36			! FGA Group
	ldd		[%o1 + 0x070], %f12		! MS
	fmovd		%f6, %f38			! FGA Group
	add		%o0, 0x40, %o0			! A0
	ldd		[%o1 + 0x078], %f14		! AX
	fmovd		%f8, %f40			! FGA Group
	fmovd		%f10, %f42			! FGA Group
	fmovd		%f12, %f44			! FGA Group
	fmovd		%f14, %f46			! FGA Group
	stda		%f32, [%o0] ASI_BLK_P		! MS
	.globl		cheetah_copy_user_page_nop_2_3
cheetah_copy_user_page_nop_2_3:
	mov		PRIMARY_CONTEXT, %o2
	stxa		%g0, [%o2] ASI_DMMU		! Flush P-cache
	stxa		%g3, [%g0] ASI_DCU_CONTROL_REG	! Disable P-cache
	ba,a,pt		%xcc, copy_user_page_continue

spitfire_copy_user_page:
	ldda		[%o1] ASI_BLK_P, %f0
	add		%o1, 0x40, %o1
	ldda		[%o1] ASI_BLK_P, %f16
	add		%o1, 0x40, %o1
	sethi		%hi(PAGE_SIZE), %o2
1:	TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
	ldda		[%o1] ASI_BLK_P, %f32
	stda		%f48, [%o0] ASI_BLK_P
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
	ldda		[%o1] ASI_BLK_P, %f0
	stda		%f48, [%o0] ASI_BLK_P
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
	ldda		[%o1] ASI_BLK_P, %f16
	stda		%f48, [%o0] ASI_BLK_P
	sub		%o2, 0x40, %o2
	add		%o1, 0x40, %o1
	cmp		%o2, PAGE_SIZE_REM
	bne,pt		%xcc, 1b
	 add		%o0, 0x40, %o0
#if (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
	TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
	ldda		[%o1] ASI_BLK_P, %f32
	stda		%f48, [%o0] ASI_BLK_P
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
	ldda		[%o1] ASI_BLK_P, %f0
	stda		%f48, [%o0] ASI_BLK_P
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	membar		#Sync
	stda		%f32, [%o0] ASI_BLK_P
	add		%o0, 0x40, %o0
	stda		%f0, [%o0] ASI_BLK_P
#else
	membar		#Sync
	stda		%f0, [%o0] ASI_BLK_P
	add		%o0, 0x40, %o0
	stda		%f16, [%o0] ASI_BLK_P
#endif
copy_user_page_continue:
	membar		#Sync
	VISExit

	mov		TLB_TAG_ACCESS, %o2
	stxa		%g5, [%o2] ASI_DMMU
	stxa		%g7, [%o3] ASI_DTLB_DATA_ACCESS
	membar		#Sync
	sub		%o3, (TLBTEMP_ENTSZ), %o3
	stxa		%o4, [%o2] ASI_DMMU
	stxa		%o5, [%o3] ASI_DTLB_DATA_ACCESS
	membar		#Sync
	rdpr		%pstate, %g3
	jmpl		%o7 + 0x8, %g0
	 wrpr		%g3, PSTATE_IE, %pstate

copy_page_using_blkcommit:
	membar		#LoadStore | #StoreStore | #StoreLoad
	ldda		[%o1] ASI_BLK_P, %f0
	add		%o1, 0x40, %o1
	ldda		[%o1] ASI_BLK_P, %f16
	add		%o1, 0x40, %o1
	sethi		%hi(PAGE_SIZE), %o2
1:	TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
	ldda		[%o1] ASI_BLK_P, %f32
	stda		%f48, [%o0] ASI_BLK_COMMIT_P
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
	ldda		[%o1] ASI_BLK_P, %f0
	stda		%f48, [%o0] ASI_BLK_COMMIT_P
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
	ldda		[%o1] ASI_BLK_P, %f16
	stda		%f48, [%o0] ASI_BLK_COMMIT_P
	sub		%o2, 0x40, %o2
	add		%o1, 0x40, %o1
	cmp		%o2, PAGE_SIZE_REM
	bne,pt		%xcc, 1b
	 add		%o0, 0x40, %o0
#if (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
	TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
	ldda		[%o1] ASI_BLK_P, %f32
	stda		%f48, [%o0] ASI_BLK_COMMIT_P
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
	ldda		[%o1] ASI_BLK_P, %f0
	stda		%f48, [%o0] ASI_BLK_COMMIT_P
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	membar		#Sync
	stda		%f32, [%o0] ASI_BLK_COMMIT_P
	add		%o0, 0x40, %o0
	ba,pt		%xcc, copy_user_page_continue
	 stda		%f0, [%o0] ASI_BLK_COMMIT_P
#else
	membar		#Sync
	stda		%f0, [%o0] ASI_BLK_COMMIT_P
	add		%o0, 0x40, %o0
	ba,pt		%xcc, copy_user_page_continue
	 stda		%f16, [%o0] ASI_BLK_COMMIT_P
#endif

	.align		32
	.globl		_clear_page
	.type		_clear_page,@function
_clear_page:	/* %o0=dest */
	VISEntryHalf
	ba,pt		%xcc, clear_page_common
	 clr		%o4

	.align		32
	.globl		clear_user_page
	.type		clear_user_page,@function
clear_user_page:	/* %o0=dest, %o1=vaddr */
	VISEntryHalf
	sethi		%hi(PAGE_SIZE), %g3
	sub		%o0, %g4, %g1
	and		%o1, %g3, %o0
	mov		TLB_TAG_ACCESS, %o2
	sethi		%uhi(_PAGE_VALID | _PAGE_SZBITS), %g3
	sethi		%hi(TLBTEMP_BASE), %o3
	sllx		%g3, 32, %g3
	or		%g3, (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), %g3
	or		%g1, %g3, %g1
	add		%o0, %o3, %o0
#define FIX_INSN_2	0x96102068 /* mov (13 << 3), %o3 */
cheetah_patch_2:
	mov		TLBTEMP_ENT2, %o3
	rdpr		%pstate, %g3
	wrpr		%g3, PSTATE_IE, %pstate

	/* Spitfire Errata #32 workaround */
	mov		PRIMARY_CONTEXT, %g5
	stxa		%g0, [%g5] ASI_DMMU
	membar		#Sync

	ldxa		[%o3] ASI_DTLB_TAG_READ, %g5

	/* Spitfire Errata #32 workaround */
	mov		PRIMARY_CONTEXT, %g7
	stxa		%g0, [%g7] ASI_DMMU
	membar		#Sync

	ldxa		[%o3] ASI_DTLB_DATA_ACCESS, %g0
	ldxa		[%o3] ASI_DTLB_DATA_ACCESS, %g7
	stxa		%o0, [%o2] ASI_DMMU
	stxa		%g1, [%o3] ASI_DTLB_DATA_ACCESS
	membar		#Sync

	mov		1, %o4

clear_page_common:
	membar		#StoreLoad | #StoreStore | #LoadStore	! LSU	Group
	fzero		%f0				! FPA	Group
	sethi		%hi(PAGE_SIZE/256), %o1		! IEU0
	fzero		%f2				! FPA	Group
	or		%o1, %lo(PAGE_SIZE/256), %o1	! IEU0
	faddd		%f0, %f2, %f4			! FPA	Group
	fmuld		%f0, %f2, %f6			! FPM
	faddd		%f0, %f2, %f8			! FPA	Group
	fmuld		%f0, %f2, %f10			! FPM

	faddd		%f0, %f2, %f12			! FPA	Group
	fmuld		%f0, %f2, %f14			! FPM
1:	stda		%f0, [%o0 + %g0] ASI_BLK_P	! Store	Group
	add		%o0, 0x40, %o0			! IEU0
	stda		%f0, [%o0 + %g0] ASI_BLK_P	! Store	Group
	add		%o0, 0x40, %o0			! IEU0
	stda		%f0, [%o0 + %g0] ASI_BLK_P	! Store	Group

	add		%o0, 0x40, %o0			! IEU0	Group
	stda		%f0, [%o0 + %g0] ASI_BLK_P	! Store	Group
	subcc		%o1, 1, %o1			! IEU1
	bne,pt		%icc, 1b			! CTI
	 add		%o0, 0x40, %o0			! IEU0	Group
	membar		#Sync				! LSU	Group
	VISExitHalf

	brnz,pt		%o4, 1f
	 nop

	retl
	 nop

1:
	stxa		%g5, [%o2] ASI_DMMU
	stxa		%g7, [%o3] ASI_DTLB_DATA_ACCESS
	membar		#Sync
	jmpl		%o7 + 0x8, %g0
	 wrpr		%g3, 0x0, %pstate

	.globl		cheetah_patch_pgcopyops
cheetah_patch_pgcopyops:
	sethi		%hi(FIX_INSN_1), %g1
	or		%g1, %lo(FIX_INSN_1), %g1
	sethi		%hi(cheetah_patch_1), %g2
	or		%g2, %lo(cheetah_patch_1), %g2
	stw		%g1, [%g2]
	flush		%g2
	sethi		%hi(FIX_INSN_2), %g1
	or		%g1, %lo(FIX_INSN_2), %g1
	sethi		%hi(cheetah_patch_2), %g2
	or		%g2, %lo(cheetah_patch_2), %g2
	stw		%g1, [%g2]
	flush		%g2
	retl
	 nop

#undef FIX_INSN1
#undef FIX_INSN2
#undef PAGE_SIZE_REM