memcpy.s

Linux内核源代码 为压缩文件 是<<Linux内核>>一书中的源代码

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Unified implementation of memcpy, memmove and the __copy_user backend.
 *
 * Copyright (C) 1998, 1999, 2000 Ralf Baechle
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 *
 * For __rmemcpy and memmove an exception is always a kernel bug, therefore
 * they're not protected.  In order to keep the exception fixup routine
 * simple all memory accesses in __copy_user to src rsp. dst are stricly
 * incremental.  The fixup routine depends on $at not being changed.
 */
#include <asm/asm.h>
#include <asm/offset.h>
#include <asm/regdef.h>

/*
 * The fixup routine for copy_to_user depends on copying strictly in
 * increasing order.  Gas expands the ulw/usw macros in the wrong order for
 * little endian machines, so we cannot depend on them.
 */
#ifdef __MIPSEB__
#define uswL	swl
#define uswU	swr
#define ulwL	lwl
#define ulwU	lwr
#define usdL	sdl
#define usdU	sdr
#define uldL	ldl
#define uldU	ldr
#endif
#ifdef __MIPSEL__
#define uswL	swr
#define uswU	swl
#define ulwL	lwr
#define ulwU	lwl
#define usdL	sdr
#define usdU	sdl
#define uldL	ldr
#define uldU	ldl
#endif

#define EX(insn,reg,addr,handler)			\
9:	insn	reg, addr;				\
	.section __ex_table,"a"; 			\
	PTR	9b, handler; 				\
	.previous

#define UEX(insn,reg,addr,handler)			\
9:	insn ## L reg, addr;				\
10:	insn ## U reg, 3 + addr;			\
	.section __ex_table,"a"; 			\
	PTR	9b, handler; 				\
	PTR	10b, handler; 				\
	.previous

#define UEXD(insn,reg,addr,handler)			\
9:	insn ## L reg, addr;				\
10:	insn ## U reg, 7 + addr;			\
	.section __ex_table,"a"; 			\
	PTR	9b, handler; 				\
	PTR	10b, handler; 				\
	.previous

/* ascending order, destination aligned  */
#define MOVE_BIGGERCHUNK(src, dst, offset, t0, t1, t2, t3) \
	EX(ld, t0, (offset + 0x00)(src), l_fixup); \
	EX(ld, t1, (offset + 0x08)(src), l_fixup); \
	EX(ld, t2, (offset + 0x10)(src), l_fixup); \
	EX(ld, t3, (offset + 0x18)(src), l_fixup); \
	EX(sd, t0, (offset + 0x00)(dst), s_fixup); \
	EX(sd, t1, (offset + 0x08)(dst), s_fixup); \
	EX(sd, t2, (offset + 0x10)(dst), s_fixup); \
	EX(sd, t3, (offset + 0x18)(dst), s_fixup); \
	EX(ld, t0, (offset + 0x20)(src), l_fixup); \
	EX(ld, t1, (offset + 0x28)(src), l_fixup); \
	EX(ld, t2, (offset + 0x30)(src), l_fixup); \
	EX(ld, t3, (offset + 0x38)(src), l_fixup); \
	EX(sd, t0, (offset + 0x20)(dst), s_fixup); \
	EX(sd, t1, (offset + 0x28)(dst), s_fixup); \
	EX(sd, t2, (offset + 0x30)(dst), s_fixup); \
	EX(sd, t3, (offset + 0x38)(dst), s_fixup)

/* ascending order, destination aligned  */
#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3) \
	EX(lw, t0, (offset + 0x00)(src), l_fixup); \
	EX(lw, t1, (offset + 0x04)(src), l_fixup); \
	EX(lw, t2, (offset + 0x08)(src), l_fixup); \
	EX(lw, t3, (offset + 0x0c)(src), l_fixup); \
	EX(sw, t0, (offset + 0x00)(dst), s_fixup); \
	EX(sw, t1, (offset + 0x04)(dst), s_fixup); \
	EX(sw, t2, (offset + 0x08)(dst), s_fixup); \
	EX(sw, t3, (offset + 0x0c)(dst), s_fixup); \
	EX(lw, t0, (offset + 0x10)(src), l_fixup); \
	EX(lw, t1, (offset + 0x14)(src), l_fixup); \
	EX(lw, t2, (offset + 0x18)(src), l_fixup); \
	EX(lw, t3, (offset + 0x1c)(src), l_fixup); \
	EX(sw, t0, (offset + 0x10)(dst), s_fixup); \
	EX(sw, t1, (offset + 0x14)(dst), s_fixup); \
	EX(sw, t2, (offset + 0x18)(dst), s_fixup); \
	EX(sw, t3, (offset + 0x1c)(dst), s_fixup)

/* ascending order, destination unaligned  */
#define UMOVE_BIGGERCHUNK(src, dst, offset, t0, t1, t2, t3) \
	EX(ld, t0, (offset + 0x00)(src), l_fixup); \
	EX(ld, t1, (offset + 0x08)(src), l_fixup); \
	EX(ld, t2, (offset + 0x10)(src), l_fixup); \
	EX(ld, t3, (offset + 0x18)(src), l_fixup); \
	UEXD(usd, t0, (offset + 0x00)(dst), s_fixup); \
	UEXD(usd, t1, (offset + 0x08)(dst), s_fixup); \
	UEXD(usd, t2, (offset + 0x10)(dst), s_fixup); \
	UEXD(usd, t3, (offset + 0x18)(dst), s_fixup); \
	EX(ld, t0, (offset + 0x20)(src), l_fixup); \
	EX(ld, t1, (offset + 0x28)(src), l_fixup); \
	EX(ld, t2, (offset + 0x30)(src), l_fixup); \
	EX(ld, t3, (offset + 0x38)(src), l_fixup); \
	UEXD(usd, t0, (offset + 0x20)(dst), s_fixup); \
	UEXD(usd, t1, (offset + 0x28)(dst), s_fixup); \
	UEXD(usd, t2, (offset + 0x30)(dst), s_fixup); \
	UEXD(usd, t3, (offset + 0x38)(dst), s_fixup)

/* ascending order, destination unaligned  */
#define UMOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3) \
	EX(lw, t0, (offset + 0x00)(src), l_fixup); \
	EX(lw, t1, (offset + 0x04)(src), l_fixup); \
	EX(lw, t2, (offset + 0x08)(src), l_fixup); \
	EX(lw, t3, (offset + 0x0c)(src), l_fixup); \
	UEX(usw, t0, (offset + 0x00)(dst), s_fixup); \
	UEX(usw, t1, (offset + 0x04)(dst), s_fixup); \
	UEX(usw, t2, (offset + 0x08)(dst), s_fixup); \
	UEX(usw, t3, (offset + 0x0c)(dst), s_fixup); \
	EX(lw, t0, (offset + 0x10)(src), l_fixup); \
	EX(lw, t1, (offset + 0x14)(src), l_fixup); \
	EX(lw, t2, (offset + 0x18)(src), l_fixup); \
	EX(lw, t3, (offset + 0x1c)(src), l_fixup); \
	UEX(usw, t0, (offset + 0x10)(dst), s_fixup); \
	UEX(usw, t1, (offset + 0x14)(dst), s_fixup); \
	UEX(usw, t2, (offset + 0x18)(dst), s_fixup); \
	UEX(usw, t3, (offset + 0x1c)(dst), s_fixup)

	.text
	.set	noreorder
	.set	noat

	.align	5
LEAF(memcpy)					/* a0=dst a1=src a2=len */
	move	v0, a0				/* return value */
__memcpy:
FEXPORT(__copy_user)
	xor	ta0, a0, a1
	andi	ta0, ta0, 0x7
	move	t3, a0
	beqz	ta0, can_align
	 sltiu	t8, a2, 0x8

	b	memcpy_u_src			# bad alignment
	 move	ta2, a2

can_align:
	bnez	t8, small_memcpy		# < 8 bytes to copy
	 move	ta2, a2

	beqz	a2, out
	 andi	t8, a1, 0x1

hword_align:
	beqz	t8, word_align
	 andi	t8, a1, 0x2

	EX(lb, ta0, (a1), l_fixup)
	dsubu	a2, a2, 0x1
	EX(sb, ta0, (a0), s_fixup)
	daddu	a1, a1, 0x1
	daddu	a0, a0, 0x1
	andi	t8, a1, 0x2

word_align:
	beqz	t8, dword_align
	 sltiu	t8, a2, 56

	EX(lh, ta0, (a1), l_fixup)
	dsubu	a2, a2, 0x2
	EX(sh, ta0, (a0), s_fixup)
	sltiu	t8, a2, 56
	daddu	a0, a0, 0x2
	daddu	a1, a1, 0x2

dword_align:
	bnez	t8, do_end_words
	 move	t8, a2

	andi	t8, a1, 0x4
	beqz	t8, qword_align
	 andi	t8, a1, 0x8

	EX(lw, ta0, 0x00(a1), l_fixup)
	dsubu	a2, a2, 0x4
	EX(sw, ta0, 0x00(a0), s_fixup)
	daddu	a1, a1, 0x4
	daddu	a0, a0, 0x4
	andi	t8, a1, 0x8

qword_align:
	beqz	t8, oword_align
	 andi	t8, a1, 0x10

	EX(lw, ta0, 0x00(a1), l_fixup)
	EX(lw, ta1, 0x04(a1), l_fixup)
	dsubu	a2, a2, 0x8
	EX(sw, ta0, 0x00(a0), s_fixup)
	EX(sw, ta1, 0x04(a0), s_fixup)
	daddu	a1, a1, 0x8
	andi	t8, a1, 0x10
	daddu	a0, a0, 0x8

oword_align:
	beqz	t8, begin_movement
	 srl	t8, a2, 0x7

	EX(lw, ta3, 0x00(a1), l_fixup)
	EX(lw, t0, 0x04(a1), l_fixup)
	EX(lw, ta0, 0x08(a1), l_fixup)
	EX(lw, ta1, 0x0c(a1), l_fixup)
	EX(sw, ta3, 0x00(a0), s_fixup)
	EX(sw, t0, 0x04(a0), s_fixup)
	EX(sw, ta0, 0x08(a0), s_fixup)
	EX(sw, ta1, 0x0c(a0), s_fixup)
	dsubu	a2, a2, 0x10
	daddu	a1, a1, 0x10
	srl	t8, a2, 0x7
	daddu	a0, a0, 0x10

begin_movement:
	beqz	t8, 0f
	 andi	ta2, a2, 0x40

move_128bytes:
	pref	0, 2*128(a0)
	pref	1, 2*128(a1)
	MOVE_BIGGERCHUNK(a1, a0, 0x00, ta0, ta1, ta3, t0)
	MOVE_BIGGERCHUNK(a1, a0, 0x40, ta0, ta1, ta3, t0)
	dsubu	t8, t8, 0x01
	daddu	a1, a1, 0x80
	bnez	t8, move_128bytes
	 daddu	a0, a0, 0x80

0:
	beqz	ta2, 1f
	 andi	ta2, a2, 0x20

move_64bytes:
	MOVE_BIGGERCHUNK(a1, a0, 0x00, ta0, ta1, ta3, t0)
	daddu	a1, a1, 0x40
	daddu	a0, a0, 0x40

1:
	beqz	ta2, do_end_words
	 andi	t8, a2, 0x1c

move_32bytes:
	MOVE_BIGCHUNK(a1, a0, 0x00, ta0, ta1, ta3, t0)
	andi	t8, a2, 0x1c
	daddu	a1, a1, 0x20
	daddu	a0, a0, 0x20

do_end_words:
	beqz	t8, maybe_end_cruft
	 srl	t8, t8, 0x2

end_words:
	EX(lw, ta0, (a1), l_fixup)
	dsubu	t8, t8, 0x1
	EX(sw, ta0, (a0), s_fixup)
	daddu	a1, a1, 0x4
	bnez	t8, end_words
	 daddu	a0, a0, 0x4

maybe_end_cruft:
	andi	ta2, a2, 0x3

small_memcpy:
	beqz	ta2, out
	 move	a2, ta2

end_bytes:
	EX(lb, ta0, (a1), l_fixup)
	dsubu	a2, a2, 0x1
	EX(sb, ta0, (a0), s_fixup)
	daddu	a1, a1, 0x1
	bnez	a2, end_bytes
	 daddu	a0, a0, 0x1

out:	jr	ra
	 move	a2, zero

/* ------------------------------------------------------------------------- */

/* Bad, bad.  At least try to align the source  */

memcpy_u_src:
	bnez	t8, small_memcpy		# < 8 bytes?
	 move	ta2, a2

	daddiu	ta0, a1, 7			# ta0: how much to align
	ori	ta0, 7
	xori	ta0, 7
	dsubu	ta0, a1

	UEXD(uld, ta1, 0(a1), l_fixup)		# dword alignment
	UEXD(usd, ta1, 0(a0), s_fixup)

	daddu	a1, ta0				# src
	daddu	a0, ta0				# dst
	dsubu	a2, ta0				# len

	sltiu	t8, a2, 56
	bnez	t8, u_do_end_words
	 andi	t8, a2, 0x3c

	andi	t8, a1, 8			# now qword aligned?

u_qword_align:
	beqz	t8, u_oword_align
	 andi	t8, a1, 0x10

	EX(ld, ta0, 0x00(a1), l_fixup)
	dsubu	a2, a2, 0x8
	UEXD(usd, ta0, 0x00(a0), s_fixup)
	daddu	a1, a1, 0x8
	andi	t8, a1, 0x10
	daddu	a0, a0, 0x8

u_oword_align:
	beqz	t8, u_begin_movement
	 srl	t8, a2, 0x7

	EX(lw, ta3, 0x08(a1), l_fixup)
	EX(lw, t0, 0x0c(a1), l_fixup)
	EX(lw, ta0, 0x00(a1), l_fixup)
	EX(lw, ta1, 0x04(a1), l_fixup)
	UEX(usw, ta3, 0x08(a0), s_fixup)
	UEX(usw, t0, 0x0c(a0), s_fixup)
	UEX(usw, ta0, 0x00(a0), s_fixup)
	UEX(usw, ta1, 0x04(a0), s_fixup)
	dsubu	a2, a2, 0x10
	daddu	a1, a1, 0x10
	srl	t8, a2, 0x7
	daddu	a0, a0, 0x10

u_begin_movement:
	beqz	t8, 0f
	 andi	ta2, a2, 0x40

u_move_128bytes:
	UMOVE_BIGGERCHUNK(a1, a0, 0x00, ta0, ta1, ta3, t0)
	UMOVE_BIGGERCHUNK(a1, a0, 0x40, ta0, ta1, ta3, t0)
	dsubu	t8, t8, 0x01
	daddu	a1, a1, 0x80
	bnez	t8, u_move_128bytes
	 daddu	a0, a0, 0x80

0:
	beqz	ta2, 1f
	 andi	ta2, a2, 0x20

u_move_64bytes:
	UMOVE_BIGGERCHUNK(a1, a0, 0x00, ta0, ta1, ta3, t0)
	daddu	a1, a1, 0x40
	daddu	a0, a0, 0x40

1:
	beqz	ta2, u_do_end_words
	 andi	t8, a2, 0x1c

u_move_32bytes: