memcpy-sh4.s

linux-2.6.15.6

.Lcase0b:
	add	#-4, r5		!  50 EX

.Lcase00b:
	mov	r0, r3		!   5 MT (latency=0)
	mov	#(~0x1f), r1	!   6 EX

	and	r1, r3		!  78 EX
	mov	r4, r2		!   5 MT (latency=0)

	cmp/eq	r3, r0		!  54 MT
	add	#0x1f, r2	!  50 EX

	bt/s	1f		! 110 BR
	 and	r1, r2		!  78 EX

	! copy initial words until cache line aligned

	mov.l	@(r0, r5), r1	!  21 LS (latency=2)
	tst	#4, r0		!  87 MT

	mov	r5, r6		!   5 MT (latency=0)
	add	#-4, r6		!  50 EX

	bt/s	4f		! 111 BR
	 add	#8, r3		!  50 EX

	tst	#0x18, r0	!  87 MT

	bt/s	1f		! 109 BR
	 mov.l	r1,@-r0		!  30 LS

	! 4 cycles, 2 long words per iteration
3:	mov.l	@(r0, r5), r1	!  21 LS (latency=2)

4:	mov.l	@(r0, r6), r7	!  21 LS (latency=2)
	cmp/eq	r3, r0		!  54 MT

	mov.l	r1, @-r0	!  30 LS
	bf/s	3b		! 109 BR

	 mov.l	r7, @-r0	!  30 LS

	! Copy the cache line aligned blocks
	!
	! In use: r0, r2, r4, r5
	! Scratch: r1, r3, r6, r7
	!
	! We could do this with the four scratch registers, but if src
	! and dest hit the same cache line, this will thrash, so make
	! use of additional registers.
	!
	! We also need r0 as a temporary (for movca), so 'undo' the invariant:
	!   r5:	 src (was r0+r5)
	!   r1:	 dest (was r0)
	! this can be reversed at the end, so we don't need to save any extra
	! state.
	!
1:	mov.l	r8, @-r15	!  30 LS
	add	r0, r5		!  49 EX

	mov.l	r9, @-r15	!  30 LS
	mov	r0, r1		!   5 MT (latency=0)

	mov.l	r10, @-r15	!  30 LS
	add	#-0x1c, r5	!  50 EX

	mov.l	r11, @-r15	!  30 LS

	! 16 cycles, 32 bytes per iteration
2:	mov.l	@(0x00,r5),r0	! 18 LS (latency=2)
	add	#-0x20, r1	! 50 EX
	mov.l	@(0x04,r5),r3	! 18 LS (latency=2)
	mov.l	@(0x08,r5),r6	! 18 LS (latency=2)
	mov.l	@(0x0c,r5),r7	! 18 LS (latency=2)
	mov.l	@(0x10,r5),r8	! 18 LS (latency=2)
	mov.l	@(0x14,r5),r9	! 18 LS (latency=2)
	mov.l	@(0x18,r5),r10	! 18 LS (latency=2)
	mov.l	@(0x1c,r5),r11	! 18 LS (latency=2)
	movca.l	r0,@r1		! 40 LS (latency=3-7)
	mov.l	r3,@(0x04,r1)	! 33 LS
	mov.l	r6,@(0x08,r1)	! 33 LS
	mov.l	r7,@(0x0c,r1)	! 33 LS

	mov.l	r8,@(0x10,r1)	! 33 LS
	add	#-0x20, r5	! 50 EX

	mov.l	r9,@(0x14,r1)	! 33 LS
	cmp/eq	r2,r1		! 54 MT

	mov.l	r10,@(0x18,r1)	!  33 LS
	bf/s	2b		! 109 BR

	 mov.l	r11,@(0x1c,r1)	!  33 LS

	mov	r1, r0		!   5 MT (latency=0)

	mov.l	@r15+, r11	!  15 LS
	sub	r1, r5		!  75 EX

	mov.l	@r15+, r10	!  15 LS
	cmp/eq	r4, r0		!  54 MT

	bf/s	1f		! 109 BR
	 mov.l	 @r15+, r9	!  15 LS

	rts
1:	 mov.l	@r15+, r8	!  15 LS
	sub	r4, r1		!  75 EX		(len remaining)

	! number of trailing bytes is non-zero
	!
	! invariants restored (r5 already decremented by 4)
	! also r1=num bytes remaining

	mov	#4, r2		!   6 EX
	mov	r4, r7		!   5 MT (latency=0)

	add	#0x1c, r5	!  50 EX		(back to -4)
	cmp/hs	r2, r1		!  58 MT

	bf/s	5f		! 108 BR
	 add	 #11, r7	!  50 EX

	mov.l	@(r0, r5), r6	!  21 LS (latency=2)
	tst	r2, r1		!  86 MT

	mov	r5, r3		!   5 MT (latency=0)
	bt/s	4f		! 111 BR

	 add	#-4, r3		!  50 EX
	cmp/hs	r2, r1		!  58 MT

	bt/s	5f		! 111 BR
	 mov.l	r6,@-r0		!  30 LS

	! 4 cycles, 2 long words per iteration
3:	mov.l	@(r0, r5), r6	!  21 LS (latency=2)

4:	mov.l	@(r0, r3), r2	!  21 LS (latency=2)
	cmp/hi	r7, r0

	mov.l	r6, @-r0	!  30 LS
	bt/s	3b		! 109 BR

	 mov.l	r2, @-r0	!  30 LS

	! Copy the final 0-3 bytes

5:	cmp/eq	r0, r4		!  54 MT
	add	#-10, r7	!  50 EX

	bt	9f		! 110 BR
	add	#3,r5		!  50 EX

	! 3 cycles, 1 byte per iteration
1:	mov.b	@(r0,r5),r1	!  19 LS
	cmp/hi	r7,r0		!  57 MT

	bt/s	1b		! 111 BR
	 mov.b	r1,@-r0		!  28 LS

9:	rts
	 nop

	!
	!	GHIJ KLMN OPQR -->  ..GH IJKL MNOP QR..
	!

	.balign	32
.Lcase2:
	! Size is 16 or greater and less then 64, but may have trailing bytes

2:	mov	r5, r6		!   5 MT (latency=0)
	add	#-2,r5		!  50 EX

	mov	r4,r2		!   5 MT (latency=0)
	add	#-4,r6		!  50 EX

	add	#7,r2		!  50 EX
3:	mov.w	@(r0,r5),r1	!  20 LS (latency=2)

	mov.w	@(r0,r6),r3	!  20 LS (latency=2)
	cmp/hi	r2,r0		!  57 MT

	mov.w	r1,@-r0		!  29 LS
	bt/s	3b		! 111 BR

	 mov.w	r3,@-r0		!  29 LS

	bra	10f
	 nop


	.balign	32
.Lcase2b:
	! Size is at least 64 bytes, so will be going round the big loop at least once.
	!
	!   r2 = rounded up r4
	!   r3 = rounded down r0

	mov	r0, r3		!   5 MT (latency=0)
	mov	#(~0x1f), r1	!   6 EX

	and	r1, r3		!  78 EX
	mov	r4, r2		!   5 MT (latency=0)

	cmp/eq	r3, r0		!  54 MT
	add	#0x1f, r2	!  50 EX

	add	#-2, r5		!  50 EX
	bt/s	1f		! 110 BR
	 and	r1, r2		!  78 EX

	! Copy a short word one at a time until we are cache line aligned
	!   Normal values: r0, r2, r3, r4
	!   Unused: r1, r6, r7
	!   Mod: r5 (=r5-2)
	!
	add	#2, r3		!  50 EX

2:	mov.w	@(r0,r5),r1	!  20 LS (latency=2)
	cmp/eq	r3,r0		!  54 MT

	bf/s	2b		! 111 BR

	 mov.w	r1,@-r0		!  29 LS

	! Copy the cache line aligned blocks
	!
	! In use: r0, r2, r4, r5 (=r5-2)
	! Scratch: r1, r3, r6, r7
	!
	! We could do this with the four scratch registers, but if src
	! and dest hit the same cache line, this will thrash, so make
	! use of additional registers.
	!
	! We also need r0 as a temporary (for movca), so 'undo' the invariant:
	!   r5:	 src (was r0+r5)
	!   r1:	 dest (was r0)
	! this can be reversed at the end, so we don't need to save any extra
	! state.
	!
1:	mov.l	r8, @-r15	!  30 LS
	add	r0, r5		!  49 EX

	mov.l	r9, @-r15	!  30 LS
	mov	r0, r1		!   5 MT (latency=0)

	mov.l	r10, @-r15	!  30 LS
	add	#-0x1e, r5	!  50 EX

	mov.l	r11, @-r15	!  30 LS

	mov.l	r12, @-r15	!  30 LS

	! 17 cycles, 32 bytes per iteration
#ifdef CONFIG_CPU_LITTLE_ENDIAN
2:	mov.w	@r5+, r0	!  14 LS (latency=2)		..JI
	add	#-0x20, r1	!  50 EX

	mov.l	@r5+, r3	!  15 LS (latency=2)		NMLK

	mov.l	@r5+, r6	!  15 LS (latency=2)		RQPO
	shll16	r0		! 103 EX			JI..

	mov.l	@r5+, r7	!  15 LS (latency=2)
	xtrct	r3, r0		!  48 EX			LKJI

	mov.l	@r5+, r8	!  15 LS (latency=2)
	xtrct	r6, r3		!  48 EX			PONM

	mov.l	@r5+, r9	!  15 LS (latency=2)
	xtrct	r7, r6		!  48 EX

	mov.l	@r5+, r10	!  15 LS (latency=2)
	xtrct	r8, r7		!  48 EX

	mov.l	@r5+, r11	!  15 LS (latency=2)
	xtrct	r9, r8		!  48 EX

	mov.w	@r5+, r12	!  15 LS (latency=2)
	xtrct	r10, r9		!  48 EX

	movca.l	r0,@r1		!  40 LS (latency=3-7)
	xtrct	r11, r10	!  48 EX

	mov.l	r3, @(0x04,r1)	!  33 LS
	xtrct	r12, r11	!  48 EX

	mov.l	r6, @(0x08,r1)	!  33 LS

	mov.l	r7, @(0x0c,r1)	!  33 LS

	mov.l	r8, @(0x10,r1)	!  33 LS
	add	#-0x40, r5	!  50 EX

	mov.l	r9, @(0x14,r1)	!  33 LS
	cmp/eq	r2,r1		!  54 MT

	mov.l	r10, @(0x18,r1)	!  33 LS
	bf/s	2b		! 109 BR

	 mov.l	r11, @(0x1c,r1)	!  33 LS
#else
2:	mov.w	@(0x1e,r5), r0	!  17 LS (latency=2)
	add	#-2, r5		!  50 EX

	mov.l	@(0x1c,r5), r3	!  18 LS (latency=2)
	add	#-4, r1		!  50 EX

	mov.l	@(0x18,r5), r6	!  18 LS (latency=2)
	shll16	r0		! 103 EX

	mov.l	@(0x14,r5), r7	!  18 LS (latency=2)
	xtrct	r3, r0		!  48 EX

	mov.l	@(0x10,r5), r8	!  18 LS (latency=2)
	xtrct	r6, r3		!  48 EX

	mov.l	@(0x0c,r5), r9	!  18 LS (latency=2)
	xtrct	r7, r6		!  48 EX

	mov.l	@(0x08,r5), r10	!  18 LS (latency=2)
	xtrct	r8, r7		!  48 EX

	mov.l	@(0x04,r5), r11	!  18 LS (latency=2)
	xtrct	r9, r8		!  48 EX

	mov.w	@(0x02,r5), r12	!  18 LS (latency=2)
	xtrct	r10, r9		!  48 EX

	movca.l	r0,@r1		!  40 LS (latency=3-7)
	add	#-0x1c, r1	!  50 EX

	mov.l	r3, @(0x1c,r1)	!  33 LS
	xtrct	r11, r10	!  48 EX

	mov.l	r6, @(0x18,r1)	!  33 LS
	xtrct	r12, r11	!  48 EX

	mov.l	r7, @(0x14,r1)	!  33 LS

	mov.l	r8, @(0x10,r1)	!  33 LS
	add	#-0x3e, r5	!  50 EX

	mov.l	r9, @(0x0c,r1)	!  33 LS
	cmp/eq	r2,r1		!  54 MT

	mov.l	r10, @(0x08,r1)	!  33 LS
	bf/s	2b		! 109 BR

	 mov.l	r11, @(0x04,r1)	!  33 LS
#endif

	mov.l	@r15+, r12
	mov	r1, r0		!   5 MT (latency=0)

	mov.l	@r15+, r11	!  15 LS
	sub	r1, r5		!  75 EX

	mov.l	@r15+, r10	!  15 LS
	cmp/eq	r4, r0		!  54 MT

	bf/s	1f		! 109 BR
	 mov.l	 @r15+, r9	!  15 LS

	rts
1:	 mov.l	@r15+, r8	!  15 LS

	add	#0x1e, r5	!  50 EX

	! Finish off a short word at a time
	! r5 must be invariant - 2
10:	mov	r4,r2		!   5 MT (latency=0)
	add	#1,r2		!  50 EX

	cmp/hi	r2, r0		!  57 MT
	bf/s	1f		! 109 BR

	 add	#2, r2		!  50 EX

3:	mov.w	@(r0,r5),r1	!  20 LS
	cmp/hi	r2,r0		!  57 MT

	bt/s	3b		! 109 BR

	 mov.w	r1,@-r0		!  29 LS
1:

	!
	! Finally, copy the last byte if necessary
	cmp/eq	r4,r0		!  54 MT
	bt/s	9b
	 add	#1,r5
	mov.b	@(r0,r5),r1
	rts
	 mov.b	r1,@-r0