gemv_n_sse2_opteron.s

Optimized GotoBLAS libraries

	addpd	 %xmm12, %xmm4
	movlpd	 8 * SIZE(A2, LDA, 2), %xmm12
	movhpd	 9 * SIZE(A2, LDA, 2), %xmm12
	mulpd	 %xmm2, %xmm8
	addpd	 %xmm13, %xmm5
	movlpd	10 * SIZE(A2, LDA, 2), %xmm13
	movhpd	11 * SIZE(A2, LDA, 2), %xmm13
	mulpd	 %xmm2, %xmm9

	addpd	 %xmm14, %xmm6
	movlpd	12 * SIZE(A2, LDA, 2), %xmm14
	movhpd	13 * SIZE(A2, LDA, 2), %xmm14
	mulpd	 %xmm2, %xmm10
	addpd	 %xmm15, %xmm7
	movlpd	14 * SIZE(A2, LDA, 2), %xmm15
	movhpd	15 * SIZE(A2, LDA, 2), %xmm15
	mulpd	 %xmm2, %xmm11

	addpd	 %xmm8,  %xmm4
	mulpd	 %xmm3, %xmm12
	addpd	 %xmm9,  %xmm5
	mulpd	 %xmm3, %xmm13

	addpd	 %xmm10, %xmm6
	mulpd	 %xmm3, %xmm14
	addpd	 %xmm11, %xmm7
	mulpd	 %xmm3, %xmm15

	addpd	 %xmm12, %xmm4
	addpd	 %xmm13, %xmm5
	addpd	 %xmm14, %xmm6
	addpd	 %xmm15, %xmm7

	movlpd	 %xmm4,  8 * SIZE(Y1)
	movhpd	 %xmm4,  9 * SIZE(Y1)
	movlpd	 %xmm5, 10 * SIZE(Y1)
	movhpd	 %xmm5, 11 * SIZE(Y1)
	movlpd	 %xmm6, 12 * SIZE(Y1)
	movhpd	 %xmm6, 13 * SIZE(Y1)
	movlpd	 %xmm7, 14 * SIZE(Y1)
	movhpd	 %xmm7, 15 * SIZE(Y1)

	subq	$-16 * SIZE, A1
	subq	$-16 * SIZE, A2
	subq	$-16 * SIZE, Y1
	ALIGN_3

.L14:
	testq	$8, M
	je	.L15

	movlpd	 0 * SIZE(A1), %xmm8
	movhpd	 1 * SIZE(A1), %xmm8
	movlpd	 2 * SIZE(A1), %xmm9
	movhpd	 3 * SIZE(A1), %xmm9
	movlpd	 4 * SIZE(A1), %xmm10
	movhpd	 5 * SIZE(A1), %xmm10
	movlpd	 6 * SIZE(A1), %xmm11
	movhpd	 7 * SIZE(A1), %xmm11

	movlpd	 0 * SIZE(A2), %xmm12
	movhpd	 1 * SIZE(A2), %xmm12
	movlpd	 2 * SIZE(A2), %xmm13
	movhpd	 3 * SIZE(A2), %xmm13
	movlpd	 4 * SIZE(A2), %xmm14
	movhpd	 5 * SIZE(A2), %xmm14
	movlpd	 6 * SIZE(A2), %xmm15
	movhpd	 7 * SIZE(A2), %xmm15

	movlpd	 0 * SIZE(Y1), %xmm4
	movhpd	 1 * SIZE(Y1), %xmm4
	movlpd	 2 * SIZE(Y1), %xmm5
	movhpd	 3 * SIZE(Y1), %xmm5
	movlpd	 4 * SIZE(Y1), %xmm6
	movhpd	 5 * SIZE(Y1), %xmm6
	movlpd	 6 * SIZE(Y1), %xmm7
	movhpd	 7 * SIZE(Y1), %xmm7

	mulpd	 %xmm0, %xmm8
	mulpd	 %xmm0, %xmm9
	mulpd	 %xmm0, %xmm10
	mulpd	 %xmm0, %xmm11

	addpd	 %xmm8,  %xmm4
	movlpd	 0 * SIZE(A1, LDA, 2), %xmm8
	movhpd	 1 * SIZE(A1, LDA, 2), %xmm8
	mulpd	 %xmm1, %xmm12
	addpd	 %xmm9,  %xmm5
	movlpd	 2 * SIZE(A1, LDA, 2), %xmm9
	movhpd	 3 * SIZE(A1, LDA, 2), %xmm9
	mulpd	 %xmm1, %xmm13

	addpd	 %xmm10, %xmm6
	movlpd	 4 * SIZE(A1, LDA, 2), %xmm10
	movhpd	 5 * SIZE(A1, LDA, 2), %xmm10
	mulpd	 %xmm1, %xmm14
	addpd	 %xmm11, %xmm7
	movlpd	 6 * SIZE(A1, LDA, 2), %xmm11
	movhpd	 7 * SIZE(A1, LDA, 2), %xmm11
	mulpd	 %xmm1, %xmm15

	addpd	 %xmm12, %xmm4
	movlpd	 0 * SIZE(A2, LDA, 2), %xmm12
	movhpd	 1 * SIZE(A2, LDA, 2), %xmm12
	mulpd	 %xmm2, %xmm8
	addpd	 %xmm13, %xmm5
	movlpd	 2 * SIZE(A2, LDA, 2), %xmm13
	movhpd	 3 * SIZE(A2, LDA, 2), %xmm13
	mulpd	 %xmm2, %xmm9

	addpd	 %xmm14, %xmm6
	movlpd	 4 * SIZE(A2, LDA, 2), %xmm14
	movhpd	 5 * SIZE(A2, LDA, 2), %xmm14
	mulpd	 %xmm2, %xmm10
	addpd	 %xmm15, %xmm7
	movlpd	 6 * SIZE(A2, LDA, 2), %xmm15
	movhpd	 7 * SIZE(A2, LDA, 2), %xmm15
	mulpd	 %xmm2, %xmm11

	addpd	 %xmm8,  %xmm4
	mulpd	 %xmm3, %xmm12
	addpd	 %xmm9,  %xmm5
	mulpd	 %xmm3, %xmm13

	addpd	 %xmm10, %xmm6
	mulpd	 %xmm3, %xmm14
	addpd	 %xmm11, %xmm7
	mulpd	 %xmm3, %xmm15

	addpd	 %xmm12, %xmm4
	mulpd	 %xmm0, %xmm8
	addpd	 %xmm13, %xmm5
	mulpd	 %xmm0, %xmm9

	addpd	 %xmm14, %xmm6
	mulpd	 %xmm0, %xmm10
	addpd	 %xmm15, %xmm7
	mulpd	 %xmm0, %xmm11

	movlpd	 %xmm4,  0 * SIZE(Y1)
	movhpd	 %xmm4,  1 * SIZE(Y1)
	movlpd	 %xmm5,  2 * SIZE(Y1)
	movhpd	 %xmm5,  3 * SIZE(Y1)
	movlpd	 %xmm6,  4 * SIZE(Y1)
	movhpd	 %xmm6,  5 * SIZE(Y1)
	movlpd	 %xmm7,  6 * SIZE(Y1)
	movhpd	 %xmm7,  7 * SIZE(Y1)

	addq	 $8 * SIZE, A1
	addq	 $8 * SIZE, A2
	addq	 $8 * SIZE, Y1
	ALIGN_3

.L15:
	testq	$4, M
	je	.L16

	movlpd	 0 * SIZE(A1), %xmm8
	movhpd	 1 * SIZE(A1), %xmm8
	movlpd	 2 * SIZE(A1), %xmm9
	movhpd	 3 * SIZE(A1), %xmm9
	movlpd	 0 * SIZE(A2), %xmm10
	movhpd	 1 * SIZE(A2), %xmm10
	movlpd	 2 * SIZE(A2), %xmm11
	movhpd	 3 * SIZE(A2), %xmm11
	movlpd	 0 * SIZE(A1, LDA, 2), %xmm12
	movhpd	 1 * SIZE(A1, LDA, 2), %xmm12
	movlpd	 2 * SIZE(A1, LDA, 2), %xmm13
	movhpd	 3 * SIZE(A1, LDA, 2), %xmm13
	movlpd	 0 * SIZE(A2, LDA, 2), %xmm14
	movhpd	 1 * SIZE(A2, LDA, 2), %xmm14
	movlpd	 2 * SIZE(A2, LDA, 2), %xmm15
	movhpd	 3 * SIZE(A2, LDA, 2), %xmm15

	movlpd	 0 * SIZE(Y1), %xmm4
	movhpd	 1 * SIZE(Y1), %xmm4
	movlpd	 2 * SIZE(Y1), %xmm5
	movhpd	 3 * SIZE(Y1), %xmm5

	mulpd	 %xmm0, %xmm8
	mulpd	 %xmm0, %xmm9
	mulpd	 %xmm1, %xmm10
	mulpd	 %xmm1, %xmm11
	mulpd	 %xmm2, %xmm12
	mulpd	 %xmm2, %xmm13
	mulpd	 %xmm3, %xmm14
	mulpd	 %xmm3, %xmm15

	addpd	 %xmm8,  %xmm4
	addpd	 %xmm9,  %xmm5
	addpd	 %xmm10, %xmm4
	addpd	 %xmm11, %xmm5
	addpd	 %xmm12, %xmm4
	addpd	 %xmm13, %xmm5
	addpd	 %xmm14, %xmm4
	addpd	 %xmm15, %xmm5

	movlpd	 %xmm4,  0 * SIZE(Y1)
	movhpd	 %xmm4,  1 * SIZE(Y1)
	movlpd	 %xmm5,  2 * SIZE(Y1)
	movhpd	 %xmm5,  3 * SIZE(Y1)

	addq	 $4 * SIZE, A1
	addq	 $4 * SIZE, A2
	addq	 $4 * SIZE, Y1
	ALIGN_3

.L16:
	testq	$2, M
	je	.L17

	movlpd	 0 * SIZE(A1), %xmm8
	movhpd	 1 * SIZE(A1), %xmm8
	movlpd	 0 * SIZE(A2), %xmm10
	movhpd	 1 * SIZE(A2), %xmm10
	movlpd	 0 * SIZE(A1, LDA, 2), %xmm12
	movhpd	 1 * SIZE(A1, LDA, 2), %xmm12
	movlpd	 0 * SIZE(A2, LDA, 2), %xmm14
	movhpd	 1 * SIZE(A2, LDA, 2), %xmm14

	movlpd	 0 * SIZE(Y1), %xmm4
	movhpd	 1 * SIZE(Y1), %xmm4

	mulpd	 %xmm0, %xmm8
	mulpd	 %xmm1, %xmm10
	mulpd	 %xmm2, %xmm12
	mulpd	 %xmm3, %xmm14

	addpd	 %xmm8,  %xmm4
	addpd	 %xmm10, %xmm4
	addpd	 %xmm12, %xmm4
	addpd	 %xmm14, %xmm4

	movlpd	 %xmm4,  0 * SIZE(Y1)
	movhpd	 %xmm4,  1 * SIZE(Y1)

	addq	 $2 * SIZE, A1
	addq	 $2 * SIZE, A2
	addq	 $2 * SIZE, Y1
	ALIGN_3

.L17:
	testq	$1, M
	je	.L19

	movlpd	 0 * SIZE(Y1), %xmm4

	movlpd	 0 * SIZE(A1), %xmm8
	movlpd	 0 * SIZE(A2), %xmm9
	movlpd	 0 * SIZE(A1, LDA, 2), %xmm10
	movlpd	 0 * SIZE(A2, LDA, 2), %xmm11

	mulsd	 %xmm0, %xmm8
	mulsd	 %xmm1, %xmm9
	mulsd	 %xmm2, %xmm10
	mulsd	 %xmm3, %xmm11

	addsd	 %xmm8,  %xmm4
	addsd	 %xmm9,  %xmm4
	addsd	 %xmm10, %xmm4
	addsd	 %xmm11, %xmm4

	movlpd	 %xmm4,  0 * SIZE(Y1)

	addq	 $1 * SIZE, A1
	addq	 $1 * SIZE, A2
	addq	 $1 * SIZE, Y1
	ALIGN_3

.L19:
	decq	J
	jg	.L11
	ALIGN_3

.L20:
	testq	$2, N
	je	.L30

	movq	YY, Y1
	movq	A,  A1
	leaq	(A, LDA, 1), A2
	leaq	(A, LDA, 2), A

	movlpd	(X), %xmm0
	addq	INCX, X
	movlpd	(X), %xmm1
	addq	INCX, X

	mulsd	STACK_ALPHA, %xmm0
	mulsd	STACK_ALPHA, %xmm1

	unpcklpd %xmm0, %xmm0
	unpcklpd %xmm1, %xmm1

	movq	M,  I
	sarq	$4, I
	jle	.L25

	movlpd	 0 * SIZE(A1), %xmm8
	movhpd	 1 * SIZE(A1), %xmm8
	movlpd	 2 * SIZE(A1), %xmm9
	movhpd	 3 * SIZE(A1), %xmm9
	movlpd	 4 * SIZE(A1), %xmm10
	movhpd	 5 * SIZE(A1), %xmm10
	movlpd	 6 * SIZE(A1), %xmm11
	movhpd	 7 * SIZE(A1), %xmm11

	movlpd	 0 * SIZE(A2), %xmm12
	movhpd	 1 * SIZE(A2), %xmm12
	movlpd	 2 * SIZE(A2), %xmm13
	movhpd	 3 * SIZE(A2), %xmm13
	movlpd	 4 * SIZE(A2), %xmm14
	movhpd	 5 * SIZE(A2), %xmm14
	movlpd	 6 * SIZE(A2), %xmm15
	movhpd	 7 * SIZE(A2), %xmm15

	movlpd	 0 * SIZE(Y1), %xmm4
	movhpd	 1 * SIZE(Y1), %xmm4
	movlpd	 2 * SIZE(Y1), %xmm5
	movhpd	 3 * SIZE(Y1), %xmm5
	movlpd	 4 * SIZE(Y1), %xmm6
	movhpd	 5 * SIZE(Y1), %xmm6
	movlpd	 6 * SIZE(Y1), %xmm7
	movhpd	 7 * SIZE(Y1), %xmm7

	mulpd	 %xmm0, %xmm8
	mulpd	 %xmm0, %xmm9
	mulpd	 %xmm0, %xmm10
	mulpd	 %xmm0, %xmm11

	decq	 I
	jle	 .L22
	ALIGN_3

.L21:
	PREFETCH	PREFETCHSIZE * SIZE(A1)
	addpd	 %xmm8,  %xmm4
	movlpd	 8 * SIZE(A1), %xmm8
	movhpd	 9 * SIZE(A1), %xmm8
	mulpd	 %xmm1, %xmm12
	addpd	 %xmm9,  %xmm5
	movlpd	10 * SIZE(A1), %xmm9
	movhpd	11 * SIZE(A1), %xmm9
	mulpd	 %xmm1, %xmm13

	addpd	 %xmm10, %xmm6
	movlpd	12 * SIZE(A1), %xmm10
	movhpd	13 * SIZE(A1), %xmm10
	mulpd	 %xmm1, %xmm14
	addpd	 %xmm11, %xmm7
	movlpd	14 * SIZE(A1), %xmm11
	movhpd	15 * SIZE(A1), %xmm11
	mulpd	 %xmm1, %xmm15

	PREFETCH	PREFETCHSIZE * SIZE(Y1)
	addpd	 %xmm12, %xmm4
	movlpd	 8 * SIZE(A2), %xmm12
	movhpd	 9 * SIZE(A2), %xmm12
	mulpd	 %xmm0, %xmm8
	addpd	 %xmm13, %xmm5
	movlpd	10 * SIZE(A2), %xmm13
	movhpd	11 * SIZE(A2), %xmm13
	mulpd	 %xmm0, %xmm9

	addpd	 %xmm14, %xmm6
	movlpd	12 * SIZE(A2), %xmm14
	movhpd	13 * SIZE(A2), %xmm14
	mulpd	 %xmm0, %xmm10
	addpd	 %xmm15, %xmm7
	movlpd	14 * SIZE(A2), %xmm15
	movhpd	15 * SIZE(A2), %xmm15
	mulpd	 %xmm0, %xmm11

	movlpd	 %xmm4, 0 * SIZE(Y1)
	movlpd	 8 * SIZE(Y1), %xmm4
	movhpd	 %xmm4, 1 * SIZE(Y1)
	movhpd	 9 * SIZE(Y1), %xmm4
	movlpd	 %xmm5, 2 * SIZE(Y1)
	movlpd	10 * SIZE(Y1), %xmm5
	movhpd	 %xmm5, 3 * SIZE(Y1)
	movhpd	11 * SIZE(Y1), %xmm5

	movlpd	 %xmm6, 4 * SIZE(Y1)
	movlpd	12 * SIZE(Y1), %xmm6
	movhpd	 %xmm6, 5 * SIZE(Y1)
 	movhpd	13 * SIZE(Y1), %xmm6
	movlpd	 %xmm7, 6 * SIZE(Y1)
	movlpd	14 * SIZE(Y1), %xmm7
	movhpd	 %xmm7, 7 * SIZE(Y1)
	movhpd	15 * SIZE(Y1), %xmm7

	PREFETCH	PREFETCHSIZE * SIZE(A2)
	addpd	 %xmm8,  %xmm4
	movlpd	16 * SIZE(A1), %xmm8
	movhpd	17 * SIZE(A1), %xmm8
	mulpd	 %xmm1, %xmm12
	addpd	 %xmm9,  %xmm5
	movlpd	18 * SIZE(A1), %xmm9
	movhpd	19 * SIZE(A1), %xmm9
	mulpd	 %xmm1, %xmm13

	addpd	 %xmm10, %xmm6
	movlpd	20 * SIZE(A1), %xmm10
	movhpd	21 * SIZE(A1), %xmm10
	mulpd	 %xmm1, %xmm14
	addpd	 %xmm11, %xmm7
	movlpd	22 * SIZE(A1), %xmm11
	movhpd	23 * SIZE(A1), %xmm11
	mulpd	 %xmm1, %xmm15

	addpd	 %xmm12, %xmm4
	movlpd	 16 * SIZE(A2), %xmm12
	movhpd	 17 * SIZE(A2), %xmm12
	mulpd	 %xmm0, %xmm8
	addpd	 %xmm13, %xmm5
	movlpd	 18 * SIZE(A2), %xmm13
	movhpd	 19 * SIZE(A2), %xmm13
	mulpd	 %xmm0, %xmm9

	addpd	 %xmm14, %xmm6
	movlpd	 20 * SIZE(A2), %xmm14
	movhpd	 21 * SIZE(A2), %xmm14
	mulpd	 %xmm0, %xmm10
	addpd	 %xmm15, %xmm7
	movlpd	 22 * SIZE(A2), %xmm15
	movhpd	 23 * SIZE(A2), %xmm15
	mulpd	 %xmm0, %xmm11

	movlpd	 %xmm4,  8 * SIZE(Y1)
	movlpd	16 * SIZE(Y1), %xmm4
	movhpd	 %xmm4,  9 * SIZE(Y1)
	movhpd	17 * SIZE(Y1), %xmm4
	movlpd	 %xmm5, 10 * SIZE(Y1)
	movlpd	18 * SIZE(Y1), %xmm5
	movhpd	 %xmm5, 11 * SIZE(Y1)
	movhpd	19 * SIZE(Y1), %xmm5
	movlpd	 %xmm6, 12 * SIZE(Y1)

	movlpd	20 * SIZE(Y1), %xmm6
	movhpd	 %xmm6, 13 * SIZE(Y1)
	movhpd	21 * SIZE(Y1), %xmm6
	movlpd	 %xmm7, 14 * SIZE(Y1)
	movlpd	22 * SIZE(Y1), %xmm7
	movhpd	 %xmm7, 15 * SIZE(Y1)
	movhpd	23 * SIZE(Y1), %xmm7

	subq	$-16 * SIZE, A1
	subq	$-16 * SIZE, A2
	subq	$-16 * SIZE, Y1
	decq	 I
	jg	.L21
	ALIGN_3

.L22:
	addpd	 %xmm8,  %xmm4
	movlpd	 8 * SIZE(A1), %xmm8
	movhpd	 9 * SIZE(A1), %xmm8
	mulpd	 %xmm1, %xmm12
	addpd	 %xmm9,  %xmm5
	movlpd	10 * SIZE(A1), %xmm9
	movhpd	11 * SIZE(A1), %xmm9
	mulpd	 %xmm1, %xmm13

	addpd	 %xmm10, %xmm6
	movlpd	12 * SIZE(A1), %xmm10
	movhpd	13 * SIZE(A1), %xmm10
	mulpd	 %xmm1, %xmm14
	addpd	 %xmm11, %xmm7
	movlpd	14 * SIZE(A1), %xmm11
	movhpd	15 * SIZE(A1), %xmm11
	mulpd	 %xmm1, %xmm15

	addpd	 %xmm12, %xmm4
	movlpd	 8 * SIZE(A2), %xmm12
	movhpd	 9 * SIZE(A2), %xmm12
	mulpd	 %xmm0, %xmm8
	addpd	 %xmm13, %xmm5
	movlpd	10 * SIZE(A2), %xmm13
	movhpd	11 * SIZE(A2), %xmm13
	mulpd	 %xmm0, %xmm9

	addpd	 %xmm14, %xmm6
	movlpd	12 * SIZE(A2), %xmm14
	movhpd	13 * SIZE(A2), %xmm14
	mulpd	 %xmm0, %xmm10
	addpd	 %xmm15, %xmm7
	movlpd	14 * SIZE(A2), %xmm15
	movhpd	15 * SIZE(A2), %xmm15
	mulpd	 %xmm0, %xmm11

	movlpd	 %xmm4, 0 * SIZE(Y1)
	movlpd	 8 * SIZE(Y1), %xmm4
	movhpd	 %xmm4, 1 * SIZE(Y1)
	movhpd	 9 * SIZE(Y1), %xmm4
	movlpd	 %xmm5, 2 * SIZE(Y1)
	movlpd	10 * SIZE(Y1), %xmm5
	movhpd	 %xmm5, 3 * SIZE(Y1)
	movhpd	11 * SIZE(Y1), %xmm5

	movlpd	 %xmm6, 4 * SIZE(Y1)
	movlpd	12 * SIZE(Y1), %xmm6
	movhpd	 %xmm6, 5 * SIZE(Y1)
 	movhpd	13 * SIZE(Y1), %xmm6
	movlpd	 %xmm7, 6 * SIZE(Y1)
	movlpd	14 * SIZE(Y1), %xmm7
	movhpd	 %xmm7, 7 * SIZE(Y1)
	movhpd	15 * SIZE(Y1), %xmm7

	addpd	 %xmm8,  %xmm4
	mulpd	 %xmm1, %xmm12
	addpd	 %xmm9,  %xmm5
	mulpd	 %xmm1, %xmm13

	addpd	 %xmm10, %xmm6
	mulpd	 %xmm1, %xmm14
	addpd	 %xmm11, %xmm7
	mulpd	 %xmm1, %xmm15

	addpd	 %xmm12, %xmm4
	addpd	 %xmm13, %xmm5
	addpd	 %xmm14, %xmm6
	addpd	 %xmm15, %xmm7

	movlpd	 %xmm4,  8 * SIZE(Y1)
	movhpd	 %xmm4,  9 * SIZE(Y1)
	movlpd	 %xmm5, 10 * SIZE(Y1)
	movhpd	 %xmm5, 11 * SIZE(Y1)
	movlpd	 %xmm6, 12 * SIZE(Y1)
	movhpd	 %xmm6, 13 * SIZE(Y1)