zgemv_n_sse2.s

Optimized GotoBLAS libraries

	addpd	 %xmm14, %xmm7
	mulpd	 %xmm1, %xmm15

	ADD	 %xmm9,  %xmm4
	mulpd	 %xmm0, %xmm8
	ADD	 %xmm11, %xmm5
	mulpd	 %xmm0, %xmm10
	ADD	 %xmm13, %xmm6
	mulpd	 %xmm0, %xmm12
	ADD	 %xmm15, %xmm7
	mulpd	 %xmm0, %xmm14

	movapd	 %xmm4,  8 * SIZE(Y1)
	movapd	 %xmm5, 10 * SIZE(Y1)
	movapd	 %xmm6, 12 * SIZE(Y1)
	movapd	 %xmm7, 14 * SIZE(Y1)

	subq	$-16 * SIZE, A1
	subq	$-16 * SIZE, Y1
	ALIGN_3

.L36:
	testq	$4, M
	je	.L37

	MOVDDUP( 0 * SIZE, A1, %xmm8)
	MOVDDUP( 1 * SIZE, A1, %xmm9)
	MOVDDUP( 2 * SIZE, A1, %xmm10)
	MOVDDUP( 3 * SIZE, A1, %xmm11)
	MOVDDUP( 4 * SIZE, A1, %xmm12)
	MOVDDUP( 5 * SIZE, A1, %xmm13)
	MOVDDUP( 6 * SIZE, A1, %xmm14)
	MOVDDUP( 7 * SIZE, A1, %xmm15)

	movapd	 0 * SIZE(Y1), %xmm4
	movapd	 2 * SIZE(Y1), %xmm5
	movapd	 4 * SIZE(Y1), %xmm6
	movapd	 6 * SIZE(Y1), %xmm7

	mulpd	 %xmm0, %xmm8
	mulpd	 %xmm0, %xmm10
	mulpd	 %xmm0, %xmm12
	mulpd	 %xmm0, %xmm14

	addpd	 %xmm8,  %xmm4
	mulpd	 %xmm1, %xmm9
	addpd	 %xmm10, %xmm5
	mulpd	 %xmm1, %xmm11

	addpd	 %xmm12, %xmm6
	mulpd	 %xmm1, %xmm13
	addpd	 %xmm14, %xmm7
	mulpd	 %xmm1, %xmm15

	ADD	 %xmm9,  %xmm4
	ADD	 %xmm11, %xmm5
	ADD	 %xmm13, %xmm6
	ADD	 %xmm15, %xmm7

	movapd	 %xmm4,  0 * SIZE(Y1)
	movapd	 %xmm5,  2 * SIZE(Y1)
	movapd	 %xmm6,  4 * SIZE(Y1)
	movapd	 %xmm7,  6 * SIZE(Y1)

	addq	 $8 * SIZE, A1
	addq	 $8 * SIZE, Y1
	ALIGN_3

.L37:
	testq	$2, M
	je	.L38

	MOVDDUP( 0 * SIZE, A1, %xmm8)
	MOVDDUP( 1 * SIZE, A1, %xmm9)
	MOVDDUP( 2 * SIZE, A1, %xmm10)
	MOVDDUP( 3 * SIZE, A1, %xmm11)

	movapd	 0 * SIZE(Y1), %xmm4
	movapd	 2 * SIZE(Y1), %xmm5

	mulpd	 %xmm0, %xmm8
	mulpd	 %xmm1, %xmm9
	mulpd	 %xmm0, %xmm10
	mulpd	 %xmm1, %xmm11

	addpd	 %xmm8,  %xmm4
	addpd	 %xmm10, %xmm5
	ADD	 %xmm9,  %xmm4
	ADD	 %xmm11, %xmm5

	movapd	 %xmm4,  0 * SIZE(Y1)
	movapd	 %xmm5,  2 * SIZE(Y1)

	addq	 $4 * SIZE, A1
	addq	 $4 * SIZE, Y1
	ALIGN_3

.L38:
	testq	$1, M
	je	.L995

	MOVDDUP( 0 * SIZE, A1, %xmm8)
	MOVDDUP( 1 * SIZE, A1, %xmm9)

	movapd	 0 * SIZE(Y1), %xmm4

	mulpd	 %xmm0, %xmm8
	mulpd	 %xmm1, %xmm9

	addpd	 %xmm8,  %xmm4
	ADD	 %xmm9,  %xmm4

	movapd	 %xmm4,  0 * SIZE(Y1)
	jmp	 .L995
	ALIGN_3

.L40:
#endif

	movq	N,  J
	sarq	$2, J
	jle	.L50
	ALIGN_3

.L41:
	movq	YY, Y1
	movq	A,  A1
	leaq	(A, LDA, 1), A2
	leaq	(A, LDA, 4), A

	MOVDDUP(0 * SIZE, X, %xmm0)
	MOVDDUP(1 * SIZE, X, %xmm1)
	addq	INCX, X
	MOVDDUP(0 * SIZE, X, %xmm2)
	MOVDDUP(1 * SIZE, X, %xmm3)
	addq	INCX, X
	MOVDDUP(0 * SIZE, X, %xmm4)
	MOVDDUP(1 * SIZE, X, %xmm5)
	addq	INCX, X
	MOVDDUP(0 * SIZE, X, %xmm6)
	MOVDDUP(1 * SIZE, X, %xmm7)
	addq	INCX, X

	movlpd	  0 + ALPHA_R,   %xmm13
	movhpd	  8 + ALPHA_R,   %xmm13
	movlpd	  0 + ALPHA_I,   %xmm14
	movhpd	  8 + ALPHA_I,   %xmm14
	movlpd	  0 + COMP_MASK, %xmm15
	movhpd	  8 + COMP_MASK, %xmm15

	mulpd	  %xmm13, %xmm0
	mulpd	  %xmm14, %xmm1
	mulpd	  %xmm13, %xmm2
	mulpd	  %xmm14, %xmm3
	mulpd	  %xmm13, %xmm4
	mulpd	  %xmm14, %xmm5
	mulpd	  %xmm13, %xmm6
	mulpd	  %xmm14, %xmm7

	ADDX	  %xmm1,  %xmm0
	ADDX	  %xmm3,  %xmm2
	ADDX	  %xmm5,  %xmm4
	ADDX	  %xmm7,  %xmm6

	movapd	  %xmm0, %xmm1
	movapd	  %xmm2, %xmm3
	movapd	  %xmm4, %xmm5
	movapd	  %xmm6, %xmm7

	SHUFPD_1 %xmm1, %xmm1
	SHUFPD_1 %xmm3, %xmm3
	SHUFPD_1 %xmm5, %xmm5
	SHUFPD_1 %xmm7, %xmm7

	xorpd	  %xmm15, %xmm1
	xorpd	  %xmm15, %xmm3
	xorpd	  %xmm15, %xmm5
	xorpd	  %xmm15, %xmm7

	movq	M,  I
	sarq	$3, I
	jle	.L44

	MOVDDUP( 0 * SIZE, A1, %xmm12)
	MOVDDUP( 2 * SIZE, A1, %xmm13)
	MOVDDUP( 4 * SIZE, A1, %xmm14)
	MOVDDUP( 6 * SIZE, A1, %xmm15)

	movlpd	 0 * SIZE(Y1), %xmm8
	movhpd	 1 * SIZE(Y1), %xmm8
	movlpd	 2 * SIZE(Y1), %xmm9
	movhpd	 3 * SIZE(Y1), %xmm9
	movlpd	 4 * SIZE(Y1), %xmm10
	movhpd	 5 * SIZE(Y1), %xmm10
	movlpd	 6 * SIZE(Y1), %xmm11
	movhpd	 7 * SIZE(Y1), %xmm11

	mulpd	 %xmm0, %xmm12
	mulpd	 %xmm0, %xmm13
	mulpd	 %xmm0, %xmm14
	mulpd	 %xmm0, %xmm15

	decq	 I
	jle	 .L43
	ALIGN_3

.L42:
#if defined(OPTERON) || defined(CORE2) || defined(PENRYN)
	PREFETCH	(PREFETCHSIZE +  0) * SIZE(A1)
#endif
#ifdef PENTIUM4
	PREFETCH	(PREFETCHSIZE +  0) * SIZE(A1)
#endif
	addpd	 %xmm12, %xmm8
	MOVDDUP( 1 * SIZE, A1, %xmm12)
	mulpd	 %xmm1, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP( 3 * SIZE, A1, %xmm13)
	mulpd	 %xmm1, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP( 5 * SIZE, A1, %xmm14)
	mulpd	 %xmm1, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP( 7 * SIZE, A1, %xmm15)
	mulpd	 %xmm1, %xmm15

#ifdef OPTERON
	PREFETCHW	(PREFETCHSIZE +  0) * SIZE(Y1)
#endif
	ADD	 %xmm12, %xmm8
	MOVDDUP( 0 * SIZE, A2, %xmm12)
	mulpd	 %xmm2, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP( 2 * SIZE, A2, %xmm13)
	mulpd	 %xmm2, %xmm13
	ADD	 %xmm14, %xmm10
	MOVDDUP( 4 * SIZE, A2, %xmm14)
	mulpd	 %xmm2, %xmm14
	ADD	 %xmm15, %xmm11
	MOVDDUP( 6 * SIZE, A2, %xmm15)
	mulpd	 %xmm2, %xmm15

#if defined(OPTERON) || defined(CORE2) || defined(PENRYN)
	PREFETCH	(PREFETCHSIZE +  0) * SIZE(A2)
#endif
#ifdef PENTIUM4
	PREFETCHW	(PREFETCHSIZE +  0) * SIZE(Y1)
#endif
	addpd	 %xmm12, %xmm8
	MOVDDUP( 1 * SIZE, A2, %xmm12)
	mulpd	 %xmm3, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP( 3 * SIZE, A2, %xmm13)
	mulpd	 %xmm3, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP( 5 * SIZE, A2, %xmm14)
	mulpd	 %xmm3, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP( 7 * SIZE, A2, %xmm15)
	mulpd	 %xmm3, %xmm15

	ADD	 %xmm12, %xmm8
	MOVDDUP2( 0 * SIZE, (A1, LDA, 2), %xmm12)
	mulpd	 %xmm4, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP2( 2 * SIZE, (A1, LDA, 2), %xmm13)
	mulpd	 %xmm4, %xmm13
	ADD	 %xmm14, %xmm10
	MOVDDUP2( 4 * SIZE, (A1, LDA, 2), %xmm14)
	mulpd	 %xmm4, %xmm14
	ADD	 %xmm15, %xmm11
	MOVDDUP2( 6 * SIZE, (A1, LDA, 2), %xmm15)
	mulpd	 %xmm4, %xmm15

#if defined(OPTERON) || defined(CORE2) || defined(PENRYN)
	PREFETCH	(PREFETCHSIZE +  0) * SIZE(A1, LDA, 2)
#endif
#ifdef PENTIUM4
	PREFETCH	(PREFETCHSIZE +  0) * SIZE(A2)
#endif
	addpd	 %xmm12, %xmm8
	MOVDDUP2( 1 * SIZE, (A1, LDA, 2), %xmm12)
	mulpd	 %xmm5, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP2( 3 * SIZE, (A1, LDA, 2), %xmm13)
	mulpd	 %xmm5, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP2( 5 * SIZE, (A1, LDA, 2), %xmm14)
	mulpd	 %xmm5, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP2( 7 * SIZE, (A1, LDA, 2), %xmm15)
	mulpd	 %xmm5, %xmm15

	ADD	 %xmm12, %xmm8
	MOVDDUP2( 0 * SIZE, (A2, LDA, 2), %xmm12)
	mulpd	 %xmm6, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP2( 2 * SIZE, (A2, LDA, 2), %xmm13)
	mulpd	 %xmm6, %xmm13
	ADD	 %xmm14, %xmm10
	MOVDDUP2( 4 * SIZE, (A2, LDA, 2), %xmm14)
	mulpd	 %xmm6, %xmm14
	ADD	 %xmm15, %xmm11
	MOVDDUP2( 6 * SIZE, (A2, LDA, 2), %xmm15)
	mulpd	 %xmm6, %xmm15

#if defined(OPTERON) || defined(CORE2) || defined(PENRYN)
	PREFETCH	(PREFETCHSIZE +  0) * SIZE(A2, LDA, 2)
#endif
	addpd	 %xmm12, %xmm8
	MOVDDUP2( 1 * SIZE, (A2, LDA, 2), %xmm12)
	mulpd	 %xmm7, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP2( 3 * SIZE, (A2, LDA, 2), %xmm13)
	mulpd	 %xmm7, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP2( 5 * SIZE, (A2, LDA, 2), %xmm14)
	mulpd	 %xmm7, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP2( 7 * SIZE, (A2, LDA, 2), %xmm15)
	mulpd	 %xmm7, %xmm15

	ADD	 %xmm12, %xmm8
	MOVDDUP( 8 * SIZE, A1, %xmm12)
	mulpd	 %xmm0, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP(10 * SIZE, A1, %xmm13)
	mulpd	 %xmm0, %xmm13
	ADD	 %xmm14, %xmm10
	MOVDDUP(12 * SIZE, A1, %xmm14)
	mulpd	 %xmm0, %xmm14
	ADD	 %xmm15, %xmm11
	MOVDDUP(14 * SIZE, A1, %xmm15)
	mulpd	 %xmm0, %xmm15

	movlpd	 %xmm8,  0 * SIZE(Y1)
	movlpd	 8 * SIZE(Y1), %xmm8
	movhpd	 %xmm8,  1 * SIZE(Y1)
	movhpd	 9 * SIZE(Y1), %xmm8

	movlpd	 %xmm9,  2 * SIZE(Y1)
	movlpd	10 * SIZE(Y1), %xmm9
	movhpd	 %xmm9,  3 * SIZE(Y1)
	movhpd	11 * SIZE(Y1), %xmm9

	movlpd	 %xmm10, 4 * SIZE(Y1)
	movlpd	12 * SIZE(Y1), %xmm10
	movhpd	 %xmm10, 5 * SIZE(Y1)
	movhpd	13 * SIZE(Y1), %xmm10

	movlpd	 %xmm11, 6 * SIZE(Y1)
	movlpd	14 * SIZE(Y1), %xmm11
	movhpd	 %xmm11, 7 * SIZE(Y1)
	movhpd	15 * SIZE(Y1), %xmm11

#if defined(OPTERON) || defined(CORE2) || defined(PENRYN)
	PREFETCH	(PREFETCHSIZE +  8) * SIZE(A1)
#endif
#ifdef PENTIUM4
	PREFETCH	(PREFETCHSIZE +  0) * SIZE(A1, LDA, 2)
#endif
	addpd	 %xmm12, %xmm8
	MOVDDUP( 9 * SIZE, A1, %xmm12)
	mulpd	 %xmm1, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP(11 * SIZE, A1, %xmm13)
	mulpd	 %xmm1, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP(13 * SIZE, A1, %xmm14)
	mulpd	 %xmm1, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP(15 * SIZE, A1, %xmm15)
	mulpd	 %xmm1, %xmm15

#ifdef OPTERON
	PREFETCHW	(PREFETCHSIZE +  8) * SIZE(Y1)
#endif
	ADD	 %xmm12, %xmm8
	MOVDDUP( 8 * SIZE, A2, %xmm12)
	mulpd	 %xmm2, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP(10 * SIZE, A2, %xmm13)
	mulpd	 %xmm2, %xmm13
	ADD	 %xmm14, %xmm10
	MOVDDUP(12 * SIZE, A2, %xmm14)
	mulpd	 %xmm2, %xmm14
	ADD	 %xmm15, %xmm11
	MOVDDUP(14 * SIZE, A2, %xmm15)
	mulpd	 %xmm2, %xmm15

#if defined(OPTERON) || defined(CORE2) || defined(PENRYN)
	PREFETCH	(PREFETCHSIZE +  8) * SIZE(A2)
#endif
	addpd	 %xmm12, %xmm8
	MOVDDUP( 9 * SIZE, A2, %xmm12)
	mulpd	 %xmm3, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP(11 * SIZE, A2, %xmm13)
	mulpd	 %xmm3, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP(13 * SIZE, A2, %xmm14)
	mulpd	 %xmm3, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP(15 * SIZE, A2, %xmm15)
	mulpd	 %xmm3, %xmm15

	ADD	 %xmm12, %xmm8
	MOVDDUP2( 8 * SIZE, (A1, LDA, 2), %xmm12)
	mulpd	 %xmm4, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP2(10 * SIZE, (A1, LDA, 2), %xmm13)
	mulpd	 %xmm4, %xmm13
	ADD	 %xmm14, %xmm10
	MOVDDUP2(12 * SIZE, (A1, LDA, 2), %xmm14)
	mulpd	 %xmm4, %xmm14
	ADD	 %xmm15, %xmm11
	MOVDDUP2(14 * SIZE, (A1, LDA, 2), %xmm15)
	mulpd	 %xmm4, %xmm15

#if defined(OPTERON) || defined(CORE2) || defined(PENRYN)
	PREFETCH	(PREFETCHSIZE +  8) * SIZE(A1, LDA, 2)
#endif
#ifdef PENTIUM4
	PREFETCH	(PREFETCHSIZE +  0) * SIZE(A2, LDA, 2)
#endif
	addpd	 %xmm12, %xmm8
	MOVDDUP2( 9 * SIZE, (A1, LDA, 2), %xmm12)
	mulpd	 %xmm5, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP2(11 * SIZE, (A1, LDA, 2), %xmm13)
	mulpd	 %xmm5, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP2(13 * SIZE, (A1, LDA, 2), %xmm14)
	mulpd	 %xmm5, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP2(15 * SIZE, (A1, LDA, 2), %xmm15)
	mulpd	 %xmm5, %xmm15

	ADD	 %xmm12, %xmm8
	MOVDDUP2( 8 * SIZE, (A2, LDA, 2), %xmm12)
	mulpd	 %xmm6, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP2(10 * SIZE, (A2, LDA, 2), %xmm13)
	mulpd	 %xmm6, %xmm13
	ADD	 %xmm14, %xmm10
	MOVDDUP2(12 * SIZE, (A2, LDA, 2), %xmm14)
	mulpd	 %xmm6, %xmm14
	ADD	 %xmm15, %xmm11
	MOVDDUP2(14 * SIZE, (A2, LDA, 2), %xmm15)
	mulpd	 %xmm6, %xmm15

#if defined(OPTERON) || defined(CORE2) || defined(PENRYN)
	PREFETCH	(PREFETCHSIZE +  8) * SIZE(A2, LDA, 2)
#endif
	addpd	 %xmm12, %xmm8
	MOVDDUP2( 9 * SIZE, (A2, LDA, 2), %xmm12)
	mulpd	 %xmm7, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP2(11 * SIZE, (A2, LDA, 2), %xmm13)
	mulpd	 %xmm7, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP2(13 * SIZE, (A2, LDA, 2), %xmm14)
	mulpd	 %xmm7, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP2(15 * SIZE, (A2, LDA, 2), %xmm15)
	mulpd	 %xmm7, %xmm15

	ADD	 %xmm12, %xmm8
	MOVDDUP(16 * SIZE, A1, %xmm12)
	mulpd	 %xmm0, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP(18 * SIZE, A1, %xmm13)
	mulpd	 %xmm0, %xmm13
	ADD	 %xmm14, %xmm10
	MOVDDUP(20 * SIZE, A1, %xmm14)
	mulpd	 %xmm0, %xmm14
	ADD	 %xmm15, %xmm11
	MOVDDUP(22 * SIZE, A1, %xmm15)
	mulpd	 %xmm0, %xmm15

	movlpd	 %xmm8,   8 * SIZE(Y1)
	movlpd	16 * SIZE(Y1), %xmm8
	movhpd	 %xmm8,   9 * SIZE(Y1)
	movhpd	17 * SIZE(Y1), %xmm8
	movlpd	 %xmm9,  10 * SIZE(Y1)
	movlpd	18 * SIZE(Y1), %xmm9
	movhpd	 %xmm9,  11 * SIZE(Y1)
	movhpd	19 * SIZE(Y1), %xmm9

	movlpd	 %xmm10, 12 * SIZE(Y1)
	movlpd	20 * SIZE(Y1), %xmm10
	movhpd	 %xmm10, 13 * SIZE(Y1)
	movhpd	21 * SIZE(Y1), %xmm10
	movlpd	 %xmm11, 14 * SIZE(Y1)
	movlpd	22 * SIZE(Y1), %xmm11
	movhpd	 %xmm11, 15 * SIZE(Y1)
	movhpd	23 * SIZE(Y1), %xmm11

	subq	$-16 * SIZE, A1
	subq	$-16 * SIZE, A2
	subq	$-16 * SIZE, Y1
	decq	 I
	jg	.L42
	ALIGN_3

.L43:
	addpd	 %xmm12, %xmm8
	MOVDDUP( 1 * SIZE, A1, %xmm12)
	mulpd	 %xmm1, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP( 3 * SIZE, A1, %xmm13)
	mulpd	 %xmm1, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP( 5 * SIZE, A1, %xmm14)
	mulpd	 %xmm1, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP( 7 * SIZE, A1, %xmm15)
	mulpd	 %xmm1, %xmm15

	ADD	 %xmm12, %xmm8
	MOVDDUP( 0 * SIZE, A2, %xmm12)
	mulpd	 %xmm2, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP( 2 * SIZE, A2, %xmm13)
	mulpd	 %xmm2, %xmm13
	ADD	 %xmm14, %xmm10
	MOVDDUP( 4 * SIZE, A2, %xmm14)
	mulpd	 %xmm2, %xmm14
	ADD	 %xmm15, %xmm11
	MOVDDUP( 6 * SIZE, A2, %xmm15)
	mulpd	 %xmm2, %xmm15

	addpd	 %xmm12, %xmm8
	MOVDDUP( 1 * SIZE, A2, %xmm12)
	mulpd	 %xmm3, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP( 3 * SIZE, A2, %xmm13)
	mulpd	 %xmm3, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP( 5 * SIZE, A2, %xmm14)
	mulpd	 %xmm3, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP( 7 * SIZE, A2, %xmm15)
	mulpd	 %xmm3, %xmm15

	ADD	 %xmm12, %xmm8
	MOVDDUP2( 0 * SIZE, (A1, LDA, 2), %xmm12)
	mulpd	 %xmm4, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP2( 2 * SIZE, (A1, LDA, 2), %xmm13)
	mulpd	 %xmm4, %xmm13
	ADD	 %xmm14, %xmm10
	MOVDDUP2( 4 * SIZE, (A1, LDA, 2), %xmm14)
	mulpd	 %xmm4, %xmm14
	ADD	 %xmm15, %xmm11
	MOVDDUP2( 6 * SIZE, (A1, LDA, 2), %xmm15)
	mulpd	 %xmm4, %xmm15

	addpd	 %xmm12, %xmm8
	MOVDDUP2( 1 * SIZE, (A1, LDA, 2), %xmm12)
	mulpd	 %xmm5, %xmm12
	addpd	 %xmm13, %xmm9
	MOVDDUP2( 3 * SIZE, (A1, LDA, 2), %xmm13)
	mulpd	 %xmm5, %xmm13
	addpd	 %xmm14, %xmm10
	MOVDDUP2( 5 * SIZE, (A1, LDA, 2), %xmm14)
	mulpd	 %xmm5, %xmm14
	addpd	 %xmm15, %xmm11
	MOVDDUP2( 7 * SIZE, (A1, LDA, 2), %xmm15)
	mulpd	 %xmm5, %xmm15

	ADD	 %xmm12, %xmm8
	MOVDDUP2( 0 * SIZE, (A2, LDA, 2), %xmm12)
	mulpd	 %xmm6, %xmm12
	ADD	 %xmm13, %xmm9
	MOVDDUP2( 2 * SIZE, (A2, LDA, 2), %xmm13)
	mulpd	 %xmm6, %xmm13
	ADD	 %xmm14, %xmm10