atl_dmm8x1x120_sse2.c

来自「基于Blas CLapck的.用过的人知道是干啥的」· C语言 代码 · 共 3,266 行 · 第 1/5 页

/*
 *             Automatically Tuned Linear Algebra Software v3.8.0
 *                    (C) Copyright 2006 R. Clint Whaley
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions, and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *   3. The name of the ATLAS group or the names of its contributers may
 *      not be used to endorse or promote products derived from this
 *      software without specific written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ATLAS GROUP OR ITS CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 */
#include "atlas_asm.h"


#include "atlas_asm.h"
/*
 * This routine designed for Core2, which seems to have relatively few
 * reservation stations on both the VPUs and ld/st, so we make it so
 * we do at most 8 computations per block, and reduce outstanding ld/st
 * over similar x86-64 code for AMD archs
 */
#if !defined(MB)
   #define MB 0
#endif
#if !defined(NB)
   #define NB 0
#endif
#if !defined(KB)
   #define KB 0
#endif
#if KB == 0
   #error "KB must be compile time constant!"
#endif
/*
 *Register usage
 */
#define pA0     %rcx
#define lda     %rbx
#define lda3    %rbp
#define lda5    %rdx
#define lda7    %rdi
#define pB0     %rax
#define pC0     %rsi
#define ldb     %r8
#define ldc     %r9
#define pfA     %r10
#define MM      %r11
#define NN      %r12
#define incAn   %r13
#define incCn   %r14
#define MM0     %r15

#define rA0     %xmm0
#define rB0     %xmm1
#define rC00    %xmm2
#define rC01    %xmm3
#define rC02    %xmm4
#define rC03    %xmm5
#define rC04    %xmm6
#define rC05    %xmm7
#define rC06    %xmm8
#define rC07    %xmm9
#define rC0     %xmm10
#define rC2     %xmm11
#define rC4     %xmm12
#define rC6     %xmm13
#define rb0     %xmm14
#define BETA    %xmm15

/*
 * Prefetch defines
 */
#if 1
   #define pref2(mem) prefetcht1        mem
   #define prefB(mem) prefetcht1        mem
   #define prefC(mem) prefetcht0        mem
#else
   #define pref2(mem)
   #define prefB(mem)
   #define prefC(mem)
#endif

#if MB == 0
   #define PFAINC 64
#else
   #define PFAINC ((MB*8+MB/8-1)/(MB/8))
#endif

#ifdef DCPLX
   #define CMUL(arg_) 2*arg_
#else
   #define CMUL(arg_) arg_
#endif

/*
                      %rdi/4       %rsi/8       %rdx/12          %xmm0/16
 void ATL_USERMM(const int M, const int N, const int K, const TYPE alpha,
                       %rcx/24         %r8/28         %r9/32           8/36
                 const TYPE *A, const int lda, const TYPE *B, const int ldb,
                        %xmm1/40    16/48          24/52
                 const TYPE beta, TYPE *C, const int ldc)
*/
        .text
.global ATL_asmdecor(ATL_USERMM)
ALIGN16
ATL_asmdecor(ATL_USERMM):
/*
 *      Save callee-saved iregs
 */
        movq    %rbp, -8(%rsp)
        movq    %rbx, -16(%rsp)
        movq    %r12, -24(%rsp)
        movq    %r13, -32(%rsp)
        movq    %r14, -40(%rsp)
        movq    %r15, -48(%rsp)
#ifdef BETAX
        pshufd  $0x44, %xmm1, BETA
/*        pshufd  $0b01000100, %xmm1, BETA */
#endif
/*
 *      Setup input parameters
 */
        movq    %rdi, MM0
        movq    %rsi, NN
        movq    %r9, pB0
        movq    %r8, lda
        movslq  8(%rsp), ldb
        movq    16(%rsp), pC0
        movslq  24(%rsp), incCn
/*
 *      ldx *= sizeof; lda3 = 3*lda, lda5=5*lda, lda7=7*lda
 */
#ifndef DCPLX
        movq    incCn, ldc
#endif
        shl     $3, lda
        shl     $3, ldb
        lea     (lda,lda,2), lda3
        lea     (lda,lda,4), lda5
        lea     (lda3,lda,4), lda7
/*
 *      pA3 += 128, pB0 += 128
 */
        sub     $-128, pA0
        sub     $-128, pB0
/*
 *      incAn = lda*M*sizeof
 */
        movq    lda, incAn
        imulq   MM0, incAn
        lea     -128(pA0,incAn), pfA
/*
 *      incCn = (ldc - M)*sizeof
 */
        sub     MM0, incCn
#ifdef DCPLX
        shl     $4, incCn
#else
        shl     $3, incCn
        test    $1, ldc
        jnz     UNLOOP
        test    $15, pC0
        jnz     UNLOOP
NLOOP:
        movq    MM0, MM
        prefB(-128(pB0,ldb))
#if KB > 16
        prefB((pB0,ldb))
#endif
#if KB > 32
        prefB(128(pB0,ldb))
#endif
#if KB > 48
        prefB(256(pB0,ldb))
#endif
#if KB > 64
        prefB(384(pB0,ldb))
#endif
MLOOP:
                prefC((pC0))
/*KLOOP: */
	movapd	-128(pB0), rB0
                                        pref2((pfA))
                                        add     $PFAINC, pfA
	movapd	-128(pA0), rC00
	mulpd	rB0,rC00
	movapd	-128(pA0,lda), rC01
	mulpd	rB0,rC01
	movapd	-128(pA0,lda,2), rC02
	mulpd	rB0,rC02
	movapd	-128(pA0,lda3), rC03
	mulpd	rB0,rC03
	movapd	-128(pA0,lda,4), rC04
	mulpd	rB0,rC04
	movapd	-128(pA0,lda5), rC05
	mulpd	rB0,rC05
	movapd	-128(pA0,lda3,2), rC06
	mulpd	rB0,rC06
	movapd	-128(pA0,lda7), rC07
	mulpd	rB0,rC07
#if KB > 2
	movapd	16-128(pB0), rB0
	movapd	16-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	16-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	16-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	16-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	16-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	16-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	16-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	16-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 4
	movapd	32-128(pB0), rB0
	movapd	32-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	32-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	32-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	32-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	32-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	32-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	32-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	32-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 6
	movapd	48-128(pB0), rB0
	movapd	48-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	48-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	48-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	48-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	48-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	48-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	48-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	48-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 8
	movapd	64-128(pB0), rB0
	movapd	64-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	64-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	64-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	64-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	64-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	64-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	64-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	64-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 10
	movapd	80-128(pB0), rB0
	movapd	80-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	80-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	80-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	80-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	80-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	80-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	80-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	80-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 12
	movapd	96-128(pB0), rB0
	movapd	96-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	96-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	96-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	96-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	96-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	96-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	96-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	96-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 14
	movapd	112-128(pB0), rB0
	movapd	112-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	112-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	112-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	112-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	112-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	112-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	112-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	112-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 16
	movapd	128-128(pB0), rB0
	movapd	128-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	128-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	128-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	128-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	128-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	128-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	128-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	128-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 18
	movapd	144-128(pB0), rB0
	movapd	144-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	144-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	144-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	144-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	144-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	144-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	144-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	144-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 20
	movapd	160-128(pB0), rB0
	movapd	160-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	160-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	160-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	160-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	160-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	160-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	160-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	160-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 22
	movapd	176-128(pB0), rB0
	movapd	176-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	176-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	176-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	176-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	176-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	176-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	176-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	176-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 24
	movapd	192-128(pB0), rB0
	movapd	192-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	192-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	192-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	192-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	192-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	192-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	192-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	192-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 26
	movapd	208-128(pB0), rB0
	movapd	208-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	208-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	208-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	208-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	208-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	208-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	208-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	208-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 28
	movapd	224-128(pB0), rB0
	movapd	224-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	224-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	224-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	224-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	224-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	224-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	224-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	224-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 30
	movapd	240-128(pB0), rB0
	movapd	240-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	240-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	240-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	240-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	240-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	240-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	240-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	240-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 32
	movapd	256-128(pB0), rB0
	movapd	256-128(pA0), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC00
	movapd	256-128(pA0,lda), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC01
	movapd	256-128(pA0,lda,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC02
	movapd	256-128(pA0,lda3), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC03
	movapd	256-128(pA0,lda,4), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC04
	movapd	256-128(pA0,lda5), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC05
	movapd	256-128(pA0,lda3,2), rA0
	mulpd	rB0,rA0
	addpd	rA0,rC06
	mulpd	256-128(pA0,lda7), rB0
	addpd	rB0,rC07
#endif

#if KB > 34
	movapd	272-128(pB0), rB0
	movapd	272-128(pA0), rA0
	mulpd	rB0,rA0