atl_dmm14x1x56_sse2pabc.c

来自「基于Blas CLapck的.用过的人知道是干啥的」· C语言 代码 · 共 2,893 行 · 第 1/5 页

/*
 *             Automatically Tuned Linear Algebra Software v3.8.0
 *                    (C) Copyright 2002 R. Clint Whaley
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions, and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *   3. The name of the ATLAS group or the names of its contributers may
 *      not be used to endorse or promote products derived from this
 *      software without specific written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ATLAS GROUP OR ITS CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 */
#ifndef ATL_GAS_x8664
   #error "This kernel requires x86-64 assembly!"
#endif
#ifdef ATL_OS_SunOS
   #define ATL_DIV_NUM MB
   #define ATL_DIV_DEN 14
#endif
#include "atlas_asm.h"


#if !defined(NB) || (NB == 0)
   #error "NB must be a compile-time constant!"
#endif
#if NB != MB || NB != KB
   #error "For this kernel, MB = NB = KB required!"
#endif
#if (NB != 56)
   #error "NB must be 56!"
#endif
#if (NB/14)*14 != NB
   #error "NB must be multiple of 14!"
#endif
#ifdef DREAL
   #define CMUL(arg_) arg_
#else
   #define CMUL(arg_) 2*arg_
#endif
/*
 * Integer register usage shown be these defines
 */
#define pA      %rcx
#define pA10	%rbx
#define ldab	%rbp
#define mldab	%rdx
#define mldab5  %rax
#define pB      %rdi
#define pC      %rsi
#define incCn   %r10
#define stM     %r9
#define stN     %r11
#define pfA	%r8
#define pA5 	pA
#define pB0	pB
/*      rax     used in 32/64 conversion */

#define NBso	(NB*8)
#define NBNBso  (NB*NB*8)
#define NB2so   (NBso+NBso)
#define NB3so   (NBso+NBso+NBso)
#define NB4so   (NBso+NBso+NBso+NBso)
#define NB5so   (NBso+NBso+NBso+NBso+NBso)
#define NB6so   (NB3so+NB3so)
#define NB7so   (NB3so+NB4so)
#define NB8so   (NB4so+NB4so)
#define NB9so   (NB4so+NB5so)
#define NB10so   (NB5so+NB5so)
#define NB11so   (NB6so+NB5so)
#define NB12so   (NB7so+NB5so)
#define NB13so   (NB8so+NB5so)
#define NB14so   (NB9so+NB5so)

/*
 * SSE2 register usage shown be these defines
 */
#define rA0	%xmm0
#define rB0	%xmm1
#define rC0	%xmm2
#define rC1	%xmm3
#define rC2	%xmm4
#define rC3	%xmm5
#define rC4	%xmm6
#define rC5	%xmm7
#define rC6	%xmm8
#define rC7	%xmm9
#define rC8	%xmm10
#define rC9	%xmm11
#define rC10	%xmm12
#define rC11	%xmm13
#define rC12	%xmm14
#define rC13	%xmm15
/*
 * Prefetch defines
 */
#if 1
#define pref2(mem) prefetcht1   mem
#define prefB(mem) prefetcht0   mem
#define prefC(mem) prefetchw    mem
#else
#define pref2(mem)
#define prefB(mem)
#define prefC(mem)
#endif

	.text
.global ATL_asmdecor(ATL_USERMM)
ATL_asmdecor(ATL_USERMM):
/*
 *      Save callee-saved iregs
 */
	movq	%rbp, -8(%rsp)
	movq	%rbx, -16(%rsp)
/*
 	movq	%r12, -32(%rsp)
 	movq	%r13, -40(%rsp)
*/
#ifdef BETAX
   #define BOF -56
	unpcklpd	%xmm1, %xmm1
	movapd	%xmm1, BOF(%rsp)
#endif
/*
 *      pA already comes in right reg
 *	Initialize pB = B; pC = C; NBso = NB * sizeof;
 */
	movq	16(%rsp), pC
			prefC((pC))
			prefC(64(pC))
	movq	%r9, pB
			prefB((pB))
			prefB(64(pB))
/*
 *      stM = pA + NBNBso;  stN = pB + NBNBso;
 */
	movq	$NBNBso, pfA
	addq	pA5, pfA
			prefB(128(pB))
/*
 *      convert ldc to 64 bits, and then set incCn = (ldc - MB)*sizeof
 */
	movl	24(%rsp), %eax
	cltq
	movq	%rax, incCn
#ifdef DREAL
	movq	%rax, stM
	subq	$MB-14, incCn
	shl	$3, incCn
#else
	subq	$(NB-14), incCn
	shl	$4, incCn
			prefC(128(pC))
			prefC(192(pC))
#endif
	addq	$120, pA5
	addq	$120, pB0
	movq	$KB*8, ldab
	movq	$-KB*5*8, mldab5
	movq	$-KB*8, mldab
	subq	mldab5, pA5
	lea	KB*8(pA5, ldab,4), pA10
	movq	$NB, stN
#ifdef DREAL
	test	$1, stM
	jnz	UNLOOP
	test	$15, pC
	jnz	UNLOOP
NLOOP:
   #ifdef ATL_DivAns
	movq	$ATL_DivAns-1, stM
   #else
	movq	$MB/14-1, stM
   #endif
MLOOP:
/*
 *	rC[0-13] = pC[0-13] * beta
 */
	ALIGN16
/*KLOOP: */
#ifdef BETA1
	movapd	0-120(pA10,mldab5,2), rC0
	movapd	0-120(pB0), rB0
	mulpd	rB0, rC0
	addsd	(pC), rC0
	movapd	0-120(pA5, mldab,4), rC1
	mulpd	rB0, rC1
	addsd	CMUL(8)(pC), rC1
	movapd	0-120(pA10, mldab,8), rC2
	mulpd	rB0, rC2
	addsd	CMUL(16)(pC), rC2
	movapd	0-120(pA5, mldab,2), rC3
	mulpd	rB0, rC3
	addsd	CMUL(24)(pC), rC3
	movapd	0-120(pA5, mldab), rC4
	mulpd	rB0, rC4
	addsd	CMUL(32)(pC), rC4
	movapd	0-120(pA5), rC5
	mulpd	rB0, rC5
	addsd	CMUL(40)(pC), rC5
	movapd	0-120(pA5, ldab), rC6
	mulpd	rB0, rC6
	addsd	CMUL(48)(pC), rC6
	movapd	0-120(pA5, ldab,2), rC7
	mulpd	rB0, rC7
	addsd	CMUL(56)(pC), rC7
	movapd	0-120(pA10, mldab,2), rC8
	mulpd	rB0, rC8
	addsd	CMUL(64)(pC), rC8
	movapd	0-120(pA5,ldab,4), rC9
	mulpd	rB0, rC9
	addsd	CMUL(72)(pC), rC9
	movapd	0-120(pA10), rC10
	mulpd	rB0, rC10
	addsd	CMUL(80)(pC), rC10
	movapd	0-120(pA10,ldab), rC11
	mulpd	rB0, rC11
	addsd	CMUL(88)(pC), rC11
	movapd	0-120(pA10,ldab,2), rC12
	mulpd	rB0, rC12
	addsd	CMUL(96)(pC), rC12
	movapd	0-120(pA5,ldab,8), rC13
	mulpd	rB0, rC13
	addsd	CMUL(104)(pC), rC13
#else
	movapd	0-120(pA10,mldab5,2), rC0
	movapd	0-120(pB0), rC13
	mulpd	rC13, rC0
	movapd	0-120(pA5, mldab,4), rC1
	mulpd	rC13, rC1
	movapd	0-120(pA10, mldab,8), rC2
	mulpd	rC13, rC2
	movapd	0-120(pA5, mldab,2), rC3
	mulpd	rC13, rC3
	movapd	0-120(pA5, mldab), rC4
	mulpd	rC13, rC4
	movapd	0-120(pA5), rC5
	mulpd	rC13, rC5
	movapd	0-120(pA5, ldab), rC6
	mulpd	rC13, rC6
	movapd	0-120(pA5, ldab,2), rC7
	mulpd	rC13, rC7
	movapd	0-120(pA10, mldab,2), rC8
	mulpd	rC13, rC8
	movapd	0-120(pA5,ldab,4), rC9
	mulpd	rC13, rC9
	movapd	0-120(pA10), rC10
	mulpd	rC13, rC10
	movapd	0-120(pA10,ldab), rC11
	mulpd	rC13, rC11
	movapd	0-120(pA10,ldab,2), rC12
	mulpd	rC13, rC12
	mulpd 	0-120(pA5,ldab,8), rC13
#endif

	movapd	16-120(pA10,mldab5,2), rA0
	movapd	16-120(pB0), rB0
	mulpd	rB0, rA0
	addpd	rA0, rC0
	movapd	16-120(pA5, mldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC1
	movapd	16-120(pA10, mldab,8), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC2
	movapd	16-120(pA5, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC3
	movapd	16-120(pA5, mldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC4
	movapd	16-120(pA5), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC5
	movapd	16-120(pA5, ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC6
	movapd	16-120(pA5, ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC7
	movapd	16-120(pA10, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC8
	movapd	16-120(pA5,ldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC9
	movapd	16-120(pA10), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC10
	movapd	16-120(pA10,ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC11
	movapd	16-120(pA10,ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC12
	mulpd	16-120(pA5,ldab,8), rB0
	addpd	rB0, rC13

	movapd	32-120(pA10,mldab5,2), rA0
	movapd	32-120(pB0), rB0
	mulpd	rB0, rA0
	addpd	rA0, rC0
	movapd	32-120(pA5, mldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC1
	movapd	32-120(pA10, mldab,8), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC2
	movapd	32-120(pA5, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC3
	movapd	32-120(pA5, mldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC4
	movapd	32-120(pA5), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC5
	movapd	32-120(pA5, ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC6
	movapd	32-120(pA5, ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC7
	movapd	32-120(pA10, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC8
	movapd	32-120(pA5,ldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC9
	movapd	32-120(pA10), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC10
	movapd	32-120(pA10,ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC11
	movapd	32-120(pA10,ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC12
	mulpd	32-120(pA5,ldab,8), rB0
	addpd	rB0, rC13

	movapd	48-120(pA10,mldab5,2), rA0
	movapd	48-120(pB0), rB0
	mulpd	rB0, rA0
	addpd	rA0, rC0
	movapd	48-120(pA5, mldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC1
	movapd	48-120(pA10, mldab,8), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC2
	movapd	48-120(pA5, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC3
	movapd	48-120(pA5, mldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC4
	movapd	48-120(pA5), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC5
	movapd	48-120(pA5, ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC6
	movapd	48-120(pA5, ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC7
	movapd	48-120(pA10, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC8
	movapd	48-120(pA5,ldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC9
	movapd	48-120(pA10), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC10
	movapd	48-120(pA10,ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC11
	movapd	48-120(pA10,ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC12
	mulpd	48-120(pA5,ldab,8), rB0
	addpd	rB0, rC13

	movapd	64-120(pA10,mldab5,2), rA0
	movapd	64-120(pB0), rB0
	mulpd	rB0, rA0
	addpd	rA0, rC0
	movapd	64-120(pA5, mldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC1
	movapd	64-120(pA10, mldab,8), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC2
	movapd	64-120(pA5, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC3
	movapd	64-120(pA5, mldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC4
	movapd	64-120(pA5), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC5
	movapd	64-120(pA5, ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC6
	movapd	64-120(pA5, ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC7
	movapd	64-120(pA10, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC8
	movapd	64-120(pA5,ldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC9
	movapd	64-120(pA10), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC10
	movapd	64-120(pA10,ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC11
	movapd	64-120(pA10,ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC12
	mulpd	64-120(pA5,ldab,8), rB0
	addpd	rB0, rC13

	movapd	80-120(pA10,mldab5,2), rA0
	movapd	80-120(pB0), rB0
	mulpd	rB0, rA0
	addpd	rA0, rC0
	movapd	80-120(pA5, mldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC1
	movapd	80-120(pA10, mldab,8), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC2
	movapd	80-120(pA5, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC3
	movapd	80-120(pA5, mldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC4
	movapd	80-120(pA5), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC5
	movapd	80-120(pA5, ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC6
	movapd	80-120(pA5, ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC7
	movapd	80-120(pA10, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC8
	movapd	80-120(pA5,ldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC9
	movapd	80-120(pA10), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC10
	movapd	80-120(pA10,ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC11
	movapd	80-120(pA10,ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC12
	mulpd	80-120(pA5,ldab,8), rB0
	addpd	rB0, rC13

	movapd	96-120(pA10,mldab5,2), rA0
	movapd	96-120(pB0), rB0
	mulpd	rB0, rA0
	addpd	rA0, rC0
	movapd	96-120(pA5, mldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC1
	movapd	96-120(pA10, mldab,8), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC2
	movapd	96-120(pA5, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC3
	movapd	96-120(pA5, mldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC4
	movapd	96-120(pA5), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC5
	movapd	96-120(pA5, ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC6
	movapd	96-120(pA5, ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC7
	movapd	96-120(pA10, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC8
	movapd	96-120(pA5,ldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC9
	movapd	96-120(pA10), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC10
	movapd	96-120(pA10,ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC11
	movapd	96-120(pA10,ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC12
	mulpd	96-120(pA5,ldab,8), rB0
	addpd	rB0, rC13

	movapd	112-120(pA10,mldab5,2), rA0
	movapd	112-120(pB0), rB0
	mulpd	rB0, rA0
	addpd	rA0, rC0
	movapd	112-120(pA5, mldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC1
	movapd	112-120(pA10, mldab,8), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC2
	movapd	112-120(pA5, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC3
	movapd	112-120(pA5, mldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC4
	movapd	112-120(pA5), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC5
	movapd	112-120(pA5, ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC6
	movapd	112-120(pA5, ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC7
	movapd	112-120(pA10, mldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC8
	movapd	112-120(pA5,ldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC9
	movapd	112-120(pA10), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC10
	movapd	112-120(pA10,ldab), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC11
	movapd	112-120(pA10,ldab,2), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC12
	mulpd	112-120(pA5,ldab,8), rB0
	addpd	rB0, rC13

	movapd	128-120(pA10,mldab5,2), rA0
	movapd	128-120(pB0), rB0
	mulpd	rB0, rA0
	addpd	rA0, rC0
	movapd	128-120(pA5, mldab,4), rA0
	mulpd	rB0, rA0
	addpd	rA0, rC1