zaxpy_sse2.s

Optimized GotoBLAS libraries

	cmpq	$2 * SIZE, INCY
	jne	.L40

	movq	M,  %rax
	sarq	$3, %rax
	jle	.L35

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)
	MOVDDUP( 2 * SIZE, X, %xmm2)
	MOVDDUP( 3 * SIZE, X, %xmm3)
	MOVDDUP( 4 * SIZE, X, %xmm4)
	MOVDDUP( 5 * SIZE, X, %xmm5)
	MOVDDUP( 6 * SIZE, X, %xmm6)
	MOVDDUP( 7 * SIZE, X, %xmm7)

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8
	movlpd	 2 * SIZE(Y), %xmm9
	movhpd	 3 * SIZE(Y), %xmm9
	movlpd	 4 * SIZE(Y), %xmm10
	movhpd	 5 * SIZE(Y), %xmm10
	movlpd	 6 * SIZE(Y), %xmm11
	movhpd	 7 * SIZE(Y), %xmm11

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm14, %xmm4
	mulpd	%xmm14, %xmm6

	decq	%rax
	jle	.L32
	ALIGN_3

.L31:
	addpd	%xmm0, %xmm8
#ifdef OPTERON
	PREFETCH	(PREFETCHSIZE + 0) * SIZE(X)
	PREFETCHW	(PREFETCHSIZE + 0) * SIZE(Y)
#endif
	mulpd	%xmm15, %xmm1
	addpd	%xmm2, %xmm9
	mulpd	%xmm15, %xmm3
	addpd	%xmm4, %xmm10
	mulpd	%xmm15, %xmm5
	addpd	%xmm6, %xmm11
	mulpd	%xmm15, %xmm7

	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9
	addpd	%xmm5, %xmm10
	addpd	%xmm7, %xmm11

	MOVDDUP( 8 * SIZE, X, %xmm0)
	MOVDDUP( 9 * SIZE, X, %xmm1)
	MOVDDUP(10 * SIZE, X, %xmm2)
	MOVDDUP(11 * SIZE, X, %xmm3)
	MOVDDUP(12 * SIZE, X, %xmm4)
	MOVDDUP(13 * SIZE, X, %xmm5)
	MOVDDUP(14 * SIZE, X, %xmm6)
	MOVDDUP(15 * SIZE, X, %xmm7)

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm14, %xmm4
	mulpd	%xmm14, %xmm6

	movlpd	%xmm8,   0 * SIZE(Y)
	movhpd	%xmm8,   1 * SIZE(Y)
	movlpd	%xmm9,   2 * SIZE(Y)
	movhpd	%xmm9,   3 * SIZE(Y)
	movlpd	%xmm10,  4 * SIZE(Y)
	movhpd	%xmm10,  5 * SIZE(Y)
	movlpd	%xmm11,  6 * SIZE(Y)
	movhpd	%xmm11,  7 * SIZE(Y)

	movlpd	 8 * SIZE(Y), %xmm8
	movhpd	 9 * SIZE(Y), %xmm8
	movlpd	10 * SIZE(Y), %xmm9
	movhpd	11 * SIZE(Y), %xmm9
	movlpd	12 * SIZE(Y), %xmm10
	movhpd	13 * SIZE(Y), %xmm10
	movlpd	14 * SIZE(Y), %xmm11
	movhpd	15 * SIZE(Y), %xmm11

	addpd	%xmm0, %xmm8
#ifdef OPTERON
	PREFETCH	(PREFETCHSIZE + 8) * SIZE(X)
	PREFETCHW	(PREFETCHSIZE + 8) * SIZE(Y)
#endif
	mulpd	%xmm15, %xmm1
	addpd	%xmm2, %xmm9
	mulpd	%xmm15, %xmm3
	addpd	%xmm4, %xmm10
	mulpd	%xmm15, %xmm5
	addpd	%xmm6, %xmm11
	mulpd	%xmm15, %xmm7

	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9
	addpd	%xmm5, %xmm10
	addpd	%xmm7, %xmm11

	MOVDDUP(16 * SIZE, X, %xmm0)
	MOVDDUP(17 * SIZE, X, %xmm1)
	MOVDDUP(18 * SIZE, X, %xmm2)
	MOVDDUP(19 * SIZE, X, %xmm3)
	MOVDDUP(20 * SIZE, X, %xmm4)
	MOVDDUP(21 * SIZE, X, %xmm5)
	MOVDDUP(22 * SIZE, X, %xmm6)
	MOVDDUP(23 * SIZE, X, %xmm7)

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm14, %xmm4
	mulpd	%xmm14, %xmm6

	movlpd	%xmm8,   8 * SIZE(Y)
	movhpd	%xmm8,   9 * SIZE(Y)
	movlpd	%xmm9,  10 * SIZE(Y)
	movhpd	%xmm9,  11 * SIZE(Y)
	movlpd	%xmm10, 12 * SIZE(Y)
	movhpd	%xmm10, 13 * SIZE(Y)
	movlpd	%xmm11, 14 * SIZE(Y)
	movhpd	%xmm11, 15 * SIZE(Y)

	movlpd	16 * SIZE(Y), %xmm8
	movhpd	17 * SIZE(Y), %xmm8
	movlpd	18 * SIZE(Y), %xmm9
	movhpd	19 * SIZE(Y), %xmm9
	movlpd	20 * SIZE(Y), %xmm10
	movhpd	21 * SIZE(Y), %xmm10
	movlpd	22 * SIZE(Y), %xmm11
	movhpd	23 * SIZE(Y), %xmm11

	addq	 $16 * SIZE, X
	addq	 $16 * SIZE, Y

	decq	%rax
	jg	.L31
	ALIGN_3

.L32:
	addpd	%xmm0, %xmm8
	mulpd	%xmm15, %xmm1
	addpd	%xmm2, %xmm9
	mulpd	%xmm15, %xmm3
	addpd	%xmm4, %xmm10
	mulpd	%xmm15, %xmm5
	addpd	%xmm6, %xmm11
	mulpd	%xmm15, %xmm7

	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9
	addpd	%xmm5, %xmm10
	addpd	%xmm7, %xmm11

	MOVDDUP( 8 * SIZE, X, %xmm0)
	MOVDDUP( 9 * SIZE, X, %xmm1)
	MOVDDUP(10 * SIZE, X, %xmm2)
	MOVDDUP(11 * SIZE, X, %xmm3)
	MOVDDUP(12 * SIZE, X, %xmm4)
	MOVDDUP(13 * SIZE, X, %xmm5)
	MOVDDUP(14 * SIZE, X, %xmm6)
	MOVDDUP(15 * SIZE, X, %xmm7)

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm14, %xmm4
	mulpd	%xmm14, %xmm6

	movlpd	%xmm8,   0 * SIZE(Y)
	movhpd	%xmm8,   1 * SIZE(Y)
	movlpd	%xmm9,   2 * SIZE(Y)
	movhpd	%xmm9,   3 * SIZE(Y)
	movlpd	%xmm10,  4 * SIZE(Y)
	movhpd	%xmm10,  5 * SIZE(Y)
	movlpd	%xmm11,  6 * SIZE(Y)
	movhpd	%xmm11,  7 * SIZE(Y)

	movlpd	 8 * SIZE(Y), %xmm8
	movhpd	 9 * SIZE(Y), %xmm8
	movlpd	10 * SIZE(Y), %xmm9
	movhpd	11 * SIZE(Y), %xmm9
	movlpd	12 * SIZE(Y), %xmm10
	movhpd	13 * SIZE(Y), %xmm10
	movlpd	14 * SIZE(Y), %xmm11
	movhpd	15 * SIZE(Y), %xmm11

	addpd	%xmm0, %xmm8
	mulpd	%xmm15, %xmm1
	addpd	%xmm2, %xmm9
	mulpd	%xmm15, %xmm3
	addpd	%xmm4, %xmm10
	mulpd	%xmm15, %xmm5
	addpd	%xmm6, %xmm11
	mulpd	%xmm15, %xmm7

	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9
	addpd	%xmm5, %xmm10
	addpd	%xmm7, %xmm11

	movlpd	%xmm8,   8 * SIZE(Y)
	movhpd	%xmm8,   9 * SIZE(Y)
	movlpd	%xmm9,  10 * SIZE(Y)
	movhpd	%xmm9,  11 * SIZE(Y)
	movlpd	%xmm10, 12 * SIZE(Y)
	movhpd	%xmm10, 13 * SIZE(Y)
	movlpd	%xmm11, 14 * SIZE(Y)
	movhpd	%xmm11, 15 * SIZE(Y)

	addq	 $16 * SIZE, X
	addq	 $16 * SIZE, Y
	ALIGN_3

.L35:
	movq	M,  %rax
	andq	$4, %rax
	jle	.L36

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)
	MOVDDUP( 2 * SIZE, X, %xmm2)
	MOVDDUP( 3 * SIZE, X, %xmm3)
	MOVDDUP( 4 * SIZE, X, %xmm4)
	MOVDDUP( 5 * SIZE, X, %xmm5)
	MOVDDUP( 6 * SIZE, X, %xmm6)
	MOVDDUP( 7 * SIZE, X, %xmm7)

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8
	movlpd	 2 * SIZE(Y), %xmm9
	movhpd	 3 * SIZE(Y), %xmm9
	movlpd	 4 * SIZE(Y), %xmm10
	movhpd	 5 * SIZE(Y), %xmm10
	movlpd	 6 * SIZE(Y), %xmm11
	movhpd	 7 * SIZE(Y), %xmm11

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm14, %xmm4
	mulpd	%xmm14, %xmm6

	addpd	%xmm0, %xmm8
	mulpd	%xmm15, %xmm1
	addpd	%xmm2, %xmm9
	mulpd	%xmm15, %xmm3
	addpd	%xmm4, %xmm10
	mulpd	%xmm15, %xmm5
	addpd	%xmm6, %xmm11
	mulpd	%xmm15, %xmm7

	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9
	addpd	%xmm5, %xmm10
	addpd	%xmm7, %xmm11

	movlpd	%xmm8,   0 * SIZE(Y)
	movhpd	%xmm8,   1 * SIZE(Y)
	movlpd	%xmm9,   2 * SIZE(Y)
	movhpd	%xmm9,   3 * SIZE(Y)
	movlpd	%xmm10,  4 * SIZE(Y)
	movhpd	%xmm10,  5 * SIZE(Y)
	movlpd	%xmm11,  6 * SIZE(Y)
	movhpd	%xmm11,  7 * SIZE(Y)

	addq	 $8 * SIZE, X
	addq	 $8 * SIZE, Y
	ALIGN_3

.L36:
	movq	M,  %rax
	andq	$2, %rax
	jle	.L37

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)
	MOVDDUP( 2 * SIZE, X, %xmm2)
	MOVDDUP( 3 * SIZE, X, %xmm3)

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8
	movlpd	 2 * SIZE(Y), %xmm9
	movhpd	 3 * SIZE(Y), %xmm9

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm15, %xmm1
	mulpd	%xmm15, %xmm3

	addpd	%xmm0, %xmm8
	addpd	%xmm2, %xmm9
	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9

	movlpd	%xmm8,   0 * SIZE(Y)
	movhpd	%xmm8,   1 * SIZE(Y)
	movlpd	%xmm9,   2 * SIZE(Y)
	movhpd	%xmm9,   3 * SIZE(Y)

	addq	 $4 * SIZE, X
	addq	 $4 * SIZE, Y
	ALIGN_3

.L37:
	movq	M,  %rax
	andq	$1, %rax
	jle	.L999

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8

	mulpd	%xmm14, %xmm0
	mulpd	%xmm15, %xmm1
	addpd	%xmm0, %xmm8
	addpd	%xmm1, %xmm8

	movlpd	%xmm8,   0 * SIZE(Y)
	movhpd	%xmm8,   1 * SIZE(Y)
	jmp	.L999
	ALIGN_3

.L40:
	movq	Y, YY
	movq	M,  %rax
	sarq	$3, %rax
	jle	.L45

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm2)
	MOVDDUP( 1 * SIZE, X, %xmm3)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm4)
	MOVDDUP( 1 * SIZE, X, %xmm5)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm6)
	MOVDDUP( 1 * SIZE, X, %xmm7)
	addq	 INCX, X

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm9
	movhpd	 1 * SIZE(Y), %xmm9
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm10
	movhpd	 1 * SIZE(Y), %xmm10
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm11
	movhpd	 1 * SIZE(Y), %xmm11
	addq	 INCY, Y

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm14, %xmm4
	mulpd	%xmm14, %xmm6

	decq	%rax
	jle	.L42
	ALIGN_3

.L41:
	addpd	%xmm0, %xmm8
#ifdef OPTERON
	PREFETCH	(PREFETCHSIZE + 0) * SIZE(X)
	PREFETCHW	(PREFETCHSIZE + 0) * SIZE(Y)
#endif
	mulpd	%xmm15, %xmm1
	addpd	%xmm2, %xmm9
	mulpd	%xmm15, %xmm3
	addpd	%xmm4, %xmm10
	mulpd	%xmm15, %xmm5
	addpd	%xmm6, %xmm11
	mulpd	%xmm15, %xmm7

	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9
	addpd	%xmm5, %xmm10
	addpd	%xmm7, %xmm11

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm2)
	MOVDDUP( 1 * SIZE, X, %xmm3)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm4)
	MOVDDUP( 1 * SIZE, X, %xmm5)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm6)
	MOVDDUP( 1 * SIZE, X, %xmm7)
	addq	 INCX, X

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm14, %xmm4
	mulpd	%xmm14, %xmm6

	movlpd	%xmm8,   0 * SIZE(YY)
	movhpd	%xmm8,   1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm9,   0 * SIZE(YY)
	movhpd	%xmm9,   1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm10,  0 * SIZE(YY)
	movhpd	%xmm10,  1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm11,  0 * SIZE(YY)
	movhpd	%xmm11,  1 * SIZE(YY)
	addq	 INCY, YY

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm9
	movhpd	 1 * SIZE(Y), %xmm9
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm10
	movhpd	 1 * SIZE(Y), %xmm10
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm11
	movhpd	 1 * SIZE(Y), %xmm11
	addq	 INCY, Y

	addpd	%xmm0, %xmm8
#ifdef OPTERON
	PREFETCH	(PREFETCHSIZE + 8) * SIZE(X)
	PREFETCHW	(PREFETCHSIZE + 8) * SIZE(Y)
#endif
	mulpd	%xmm15, %xmm1
	addpd	%xmm2, %xmm9
	mulpd	%xmm15, %xmm3
	addpd	%xmm4, %xmm10
	mulpd	%xmm15, %xmm5
	addpd	%xmm6, %xmm11
	mulpd	%xmm15, %xmm7

	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9
	addpd	%xmm5, %xmm10
	addpd	%xmm7, %xmm11

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm2)
	MOVDDUP( 1 * SIZE, X, %xmm3)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm4)
	MOVDDUP( 1 * SIZE, X, %xmm5)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm6)
	MOVDDUP( 1 * SIZE, X, %xmm7)
	addq	 INCX, X

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm14, %xmm4
	mulpd	%xmm14, %xmm6

	movlpd	%xmm8,   0 * SIZE(YY)
	movhpd	%xmm8,   1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm9,   0 * SIZE(YY)
	movhpd	%xmm9,   1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm10,  0 * SIZE(YY)
	movhpd	%xmm10,  1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm11,  0 * SIZE(YY)
	movhpd	%xmm11,  1 * SIZE(YY)
	addq	 INCY, YY

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm9
	movhpd	 1 * SIZE(Y), %xmm9
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm10
	movhpd	 1 * SIZE(Y), %xmm10
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm11
	movhpd	 1 * SIZE(Y), %xmm11
	addq	 INCY, Y

	decq	%rax
	jg	.L41
	ALIGN_3

.L42:
	addpd	%xmm0, %xmm8
	mulpd	%xmm15, %xmm1
	addpd	%xmm2, %xmm9
	mulpd	%xmm15, %xmm3
	addpd	%xmm4, %xmm10
	mulpd	%xmm15, %xmm5
	addpd	%xmm6, %xmm11
	mulpd	%xmm15, %xmm7

	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9
	addpd	%xmm5, %xmm10
	addpd	%xmm7, %xmm11

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm2)
	MOVDDUP( 1 * SIZE, X, %xmm3)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm4)
	MOVDDUP( 1 * SIZE, X, %xmm5)
	addq	 INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm6)
	MOVDDUP( 1 * SIZE, X, %xmm7)
	addq	 INCX, X

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm14, %xmm4
	mulpd	%xmm14, %xmm6

	movlpd	%xmm8,   0 * SIZE(YY)
	movhpd	%xmm8,   1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm9,   0 * SIZE(YY)
	movhpd	%xmm9,   1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm10,  0 * SIZE(YY)
	movhpd	%xmm10,  1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm11,  0 * SIZE(YY)
	movhpd	%xmm11,  1 * SIZE(YY)
	addq	 INCY, YY

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm9
	movhpd	 1 * SIZE(Y), %xmm9
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm10
	movhpd	 1 * SIZE(Y), %xmm10
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm11
	movhpd	 1 * SIZE(Y), %xmm11
	addq	 INCY, Y

	addpd	%xmm0, %xmm8
	mulpd	%xmm15, %xmm1
	addpd	%xmm2, %xmm9
	mulpd	%xmm15, %xmm3
	addpd	%xmm4, %xmm10
	mulpd	%xmm15, %xmm5
	addpd	%xmm6, %xmm11
	mulpd	%xmm15, %xmm7

	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9
	addpd	%xmm5, %xmm10
	addpd	%xmm7, %xmm11

	movlpd	%xmm8,   0 * SIZE(YY)
	movhpd	%xmm8,   1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm9,   0 * SIZE(YY)
	movhpd	%xmm9,   1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm10,  0 * SIZE(YY)
	movhpd	%xmm10,  1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm11,  0 * SIZE(YY)
	movhpd	%xmm11,  1 * SIZE(YY)
	addq	 INCY, YY
	ALIGN_3

.L45:
	movq	M,  %rax
	andq	$4, %rax
	jle	.L46

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)
	addq	INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm2)
	MOVDDUP( 1 * SIZE, X, %xmm3)
	addq	INCX, X

	MOVDDUP( 0 * SIZE, X, %xmm4)
	MOVDDUP( 1 * SIZE, X, %xmm5)
	addq	INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm6)
	MOVDDUP( 1 * SIZE, X, %xmm7)
	addq	INCX, X

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm9
	movhpd	 1 * SIZE(Y), %xmm9
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm10
	movhpd	 1 * SIZE(Y), %xmm10
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm11
	movhpd	 1 * SIZE(Y), %xmm11
	addq	 INCY, Y

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm14, %xmm4
	mulpd	%xmm14, %xmm6

	addpd	%xmm0, %xmm8
	mulpd	%xmm15, %xmm1
	addpd	%xmm2, %xmm9
	mulpd	%xmm15, %xmm3
	addpd	%xmm4, %xmm10
	mulpd	%xmm15, %xmm5
	addpd	%xmm6, %xmm11
	mulpd	%xmm15, %xmm7

	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9
	addpd	%xmm5, %xmm10
	addpd	%xmm7, %xmm11

	movlpd	%xmm8,  0 * SIZE(YY)
	movhpd	%xmm8,  1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm9,  0 * SIZE(YY)
	movhpd	%xmm9,  1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm10, 0 * SIZE(YY)
	movhpd	%xmm10, 1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm11, 0 * SIZE(YY)
	movhpd	%xmm11, 1 * SIZE(YY)
	addq	 INCY, YY
	ALIGN_3

.L46:
	movq	M,  %rax
	andq	$2, %rax
	jle	.L47

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)
	addq	INCX, X
	MOVDDUP( 0 * SIZE, X, %xmm2)
	MOVDDUP( 1 * SIZE, X, %xmm3)
	addq	INCX, X

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8
	addq	 INCY, Y
	movlpd	 0 * SIZE(Y), %xmm9
	movhpd	 1 * SIZE(Y), %xmm9
	addq	 INCY, Y

	mulpd	%xmm14, %xmm0
	mulpd	%xmm14, %xmm2
	mulpd	%xmm15, %xmm1
	mulpd	%xmm15, %xmm3

	addpd	%xmm0, %xmm8
	addpd	%xmm2, %xmm9
	addpd	%xmm1, %xmm8
	addpd	%xmm3, %xmm9

	movlpd	%xmm8,   0 * SIZE(YY)
	movhpd	%xmm8,   1 * SIZE(YY)
	addq	 INCY, YY
	movlpd	%xmm9,   0 * SIZE(YY)
	movhpd	%xmm9,   1 * SIZE(YY)
	addq	 INCY, YY
	ALIGN_3

.L47:
	movq	M,  %rax
	andq	$1, %rax
	jle	.L999

	MOVDDUP( 0 * SIZE, X, %xmm0)
	MOVDDUP( 1 * SIZE, X, %xmm1)

	movlpd	 0 * SIZE(Y), %xmm8
	movhpd	 1 * SIZE(Y), %xmm8
	mulpd	%xmm14, %xmm0
	mulpd	%xmm15, %xmm1
	addpd	%xmm0, %xmm8
	addpd	%xmm1, %xmm8

	movlpd	%xmm8,   0 * SIZE(YY)
	movhpd	%xmm8,   1 * SIZE(YY)
	ALIGN_3

.L999:
	xorq	%rax, %rax

	RESTOREREGISTERS

	ret

	EPILOGUE