📄 ztrsm_kernel_lt_2x2_penryn.s
字号:
/*********************************************************************//* *//* Optimized BLAS libraries *//* By Kazushige Goto <kgoto@tacc.utexas.edu> *//* *//* Copyright (c) The University of Texas, 2005. All rights reserved. *//* UNIVERSITY EXPRESSLY DISCLAIMS ANY AND ALL WARRANTIES CONCERNING *//* THIS SOFTWARE AND DOCUMENTATION, INCLUDING ANY WARRANTIES OF *//* MERCHANTABILITY, FITNESS FOR ANY PARTICULAR PURPOSE, *//* NON-INFRINGEMENT AND WARRANTIES OF PERFORMANCE, AND ANY WARRANTY *//* THAT MIGHT OTHERWISE ARISE FROM COURSE OF DEALING OR USAGE OF *//* TRADE. NO WARRANTY IS EITHER EXPRESS OR IMPLIED WITH RESPECT TO *//* THE USE OF THE SOFTWARE OR DOCUMENTATION. *//* Under no circumstances shall University be liable for incidental, *//* special, indirect, direct or consequential damages or loss of *//* profits, interruption of business, or related expenses which may *//* arise from use of Software or Documentation, including but not *//* limited to those resulting from defects in Software and/or *//* Documentation, or loss or inaccuracy of data of any kind. *//*********************************************************************/#define ASSEMBLER#include "common.h" #define OLD_M %rdi#define OLD_N %rsi#define OLD_K %rdx#define M %r13#define N %r14#define K %r15#define A %rcx#define B %r8#define C %r9#define LDC %r10 #define I %r11#define AO %rdi#define BO %rsi#define CO1 %rbx#define CO2 %rbp#define KK %rdx#define BB %r12#ifndef WINDOWS_ABI#define STACKSIZE 128#define OLD_LDC 8 + STACKSIZE(%rsp)#define OLD_OFFSET 16 + STACKSIZE(%rsp)#define OFFSET 48(%rsp)#define J 56(%rsp)#define KKK 64(%rsp)#define AORIG 72(%rsp)#else#define STACKSIZE 256#define OLD_A 48 + STACKSIZE(%rsp)#define OLD_B 56 + STACKSIZE(%rsp)#define OLD_C 64 + STACKSIZE(%rsp)#define OLD_LDC 72 + STACKSIZE(%rsp)#define OLD_OFFSET 80 + STACKSIZE(%rsp)#define OFFSET 224(%rsp)#define J 232(%rsp)#define KKK 240(%rsp)#define AORIG 248(%rsp)#endif#define PREFETCH_R (8 * 4 + 0)#define PREFETCHSIZE (8 * 21 + 6)#define PREFETCH prefetcht0#define ADD1 addpd#define ADD2 addpd PROLOGUE PROFCODE subq $STACKSIZE, %rsp movq %rbx, 0(%rsp) movq %rbp, 8(%rsp) movq %r12, 16(%rsp) movq %r13, 24(%rsp) movq %r14, 32(%rsp) movq %r15, 40(%rsp)#ifdef WINDOWS_ABI movq %rdi, 48(%rsp) movq %rsi, 56(%rsp) movups %xmm6, 64(%rsp) movups %xmm7, 80(%rsp) movups %xmm8, 96(%rsp) movups %xmm9, 112(%rsp) movups %xmm10, 128(%rsp) movups %xmm11, 144(%rsp) movups %xmm12, 160(%rsp) movups %xmm13, 176(%rsp) movups %xmm14, 192(%rsp) movups %xmm15, 208(%rsp) movq ARG1, OLD_M movq ARG2, OLD_N movq ARG3, OLD_K movq OLD_A, A movq OLD_B, B movq OLD_C, C#endif movq OLD_M, M movq OLD_N, N movq OLD_K, K movq OLD_LDC, LDC movq OLD_OFFSET, KK subq $-16 * SIZE, A subq $-16 * SIZE, B salq $ZBASE_SHIFT, LDC movq KK, OFFSET negq KK#ifdef LN movq M, %rax salq $ZBASE_SHIFT, %rax addq %rax, C imulq K, %rax addq %rax, A#endif#ifdef RT movq N, %rax salq $ZBASE_SHIFT, %rax imulq K, %rax addq %rax, B movq N, %rax imulq LDC, %rax addq %rax, C#endif#ifdef RT movq N, KK subq OFFSET, KK#endif movq N, J sarq $1, J jle,pn .L40 ALIGN_4.L01:#if defined(LT) || defined(RN) movq A, AO#else movq A, AORIG#endif#ifdef RT movq K, %rax salq $1 + ZBASE_SHIFT, %rax subq %rax, B leaq (, LDC, 2), %rax subq %rax, C#endif movq C, CO1 leaq (C, LDC, 1), CO2#ifndef RT leaq (C, LDC, 2), C#endif#ifdef LN movq OFFSET, KK addq M, KK#endif movq K, %rax salq $ZBASE_SHIFT + 1, %rax leaq (B, %rax), BB#ifdef LT movq OFFSET, KK#endif movq M, I sarq $1, I jle,pn .L20 ALIGN_4.L11:#ifdef LN movq K, %rax salq $1 + ZBASE_SHIFT, %rax subq %rax, AORIG#endif#if defined(LN) || defined(RT) movq KK, %rax leaq (, %rax, SIZE), %rax movq AORIG, AO leaq (AO, %rax, 4), AO leaq (B, %rax, 4), BO#else movq B, BO#endif prefetcht2 -16 * SIZE(BB) subq $-8 * SIZE, BB movaps -16 * SIZE(AO), %xmm0 pxor %xmm3, %xmm3 movaps -14 * SIZE(AO), %xmm1 pxor %xmm4, %xmm4 movaps -16 * SIZE(BO), %xmm2 pxor %xmm5, %xmm5 pxor %xmm6, %xmm6#ifdef LN prefetcht0 -4 * SIZE(CO1) movapd %xmm4, %xmm8 movapd %xmm4, %xmm9 prefetcht0 -4 * SIZE(CO2)#else prefetcht0 3 * SIZE(CO1) movapd %xmm4, %xmm8 movapd %xmm4, %xmm9 prefetcht0 3 * SIZE(CO2)#endif movapd %xmm4, %xmm10 movapd %xmm4, %xmm11 movapd %xmm4, %xmm12 movapd %xmm4, %xmm13 movapd %xmm4, %xmm14 movapd %xmm4, %xmm15#if defined(LT) || defined(RN) movq KK, %rax#else movq K, %rax subq KK, %rax#endif sarq $3, %rax jle,pn .L15 ALIGN_3.L12: ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm1, %xmm4 ADD2 %xmm5, %xmm13 ADD2 %xmm6, %xmm15 movaps %xmm7, %xmm6 mulpd %xmm0, %xmm7 mulpd %xmm1, %xmm6 ADD1 %xmm2, %xmm8 movaps -12 * SIZE(BO), %xmm2 ADD1 %xmm4, %xmm10 movaps %xmm3, %xmm4 pshufd $0x4e, %xmm3, %xmm5 mulpd %xmm0, %xmm3 mulpd %xmm1, %xmm4 ADD2 %xmm7, %xmm9 ADD2 %xmm6, %xmm11 movaps %xmm5, %xmm6 mulpd %xmm0, %xmm5 movaps -12 * SIZE(AO), %xmm0 mulpd %xmm1, %xmm6 movaps -10 * SIZE(AO), %xmm1 ADD1 %xmm3, %xmm12 movaps -10 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm1, %xmm4 ADD2 %xmm5, %xmm13 ADD2 %xmm6, %xmm15 movaps %xmm7, %xmm6 mulpd %xmm0, %xmm7 mulpd %xmm1, %xmm6 ADD1 %xmm2, %xmm8 movaps -8 * SIZE(BO), %xmm2 ADD1 %xmm4, %xmm10 movaps %xmm3, %xmm4 pshufd $0x4e, %xmm3, %xmm5 mulpd %xmm0, %xmm3 mulpd %xmm1, %xmm4 ADD2 %xmm7, %xmm9 ADD2 %xmm6, %xmm11 movaps %xmm5, %xmm6 mulpd %xmm0, %xmm5 movaps -8 * SIZE(AO), %xmm0 mulpd %xmm1, %xmm6 movaps -6 * SIZE(AO), %xmm1 ADD1 %xmm3, %xmm12 movaps -6 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm1, %xmm4 ADD2 %xmm5, %xmm13 ADD2 %xmm6, %xmm15 movaps %xmm7, %xmm6 mulpd %xmm0, %xmm7 mulpd %xmm1, %xmm6 ADD1 %xmm2, %xmm8 movaps -4 * SIZE(BO), %xmm2 ADD1 %xmm4, %xmm10 movaps %xmm3, %xmm4 pshufd $0x4e, %xmm3, %xmm5 mulpd %xmm0, %xmm3 mulpd %xmm1, %xmm4 ADD2 %xmm7, %xmm9 ADD2 %xmm6, %xmm11 movaps %xmm5, %xmm6 mulpd %xmm0, %xmm5 movaps -4 * SIZE(AO), %xmm0 mulpd %xmm1, %xmm6 movaps -2 * SIZE(AO), %xmm1 ADD1 %xmm3, %xmm12 movaps -2 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm1, %xmm4 ADD2 %xmm5, %xmm13 ADD2 %xmm6, %xmm15 movaps %xmm7, %xmm6 mulpd %xmm0, %xmm7 mulpd %xmm1, %xmm6 ADD1 %xmm2, %xmm8 movaps 0 * SIZE(BO), %xmm2 ADD1 %xmm4, %xmm10 movaps %xmm3, %xmm4 pshufd $0x4e, %xmm3, %xmm5 mulpd %xmm0, %xmm3 mulpd %xmm1, %xmm4 ADD2 %xmm7, %xmm9 ADD2 %xmm6, %xmm11 movaps %xmm5, %xmm6 mulpd %xmm0, %xmm5 movaps 0 * SIZE(AO), %xmm0 mulpd %xmm1, %xmm6 movaps 2 * SIZE(AO), %xmm1 ADD1 %xmm3, %xmm12 movaps 2 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm1, %xmm4 ADD2 %xmm5, %xmm13 ADD2 %xmm6, %xmm15 movaps %xmm7, %xmm6 mulpd %xmm0, %xmm7 mulpd %xmm1, %xmm6 ADD1 %xmm2, %xmm8 movaps 4 * SIZE(BO), %xmm2 ADD1 %xmm4, %xmm10 movaps %xmm3, %xmm4 pshufd $0x4e, %xmm3, %xmm5 mulpd %xmm0, %xmm3 mulpd %xmm1, %xmm4 ADD2 %xmm7, %xmm9 ADD2 %xmm6, %xmm11 movaps %xmm5, %xmm6 mulpd %xmm0, %xmm5 movaps 4 * SIZE(AO), %xmm0 mulpd %xmm1, %xmm6 movaps 6 * SIZE(AO), %xmm1 ADD1 %xmm3, %xmm12 movaps 6 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm1, %xmm4 ADD2 %xmm5, %xmm13 ADD2 %xmm6, %xmm15 movaps %xmm7, %xmm6 mulpd %xmm0, %xmm7 mulpd %xmm1, %xmm6 ADD1 %xmm2, %xmm8 movaps 8 * SIZE(BO), %xmm2 ADD1 %xmm4, %xmm10 movaps %xmm3, %xmm4 pshufd $0x4e, %xmm3, %xmm5 mulpd %xmm0, %xmm3 mulpd %xmm1, %xmm4 ADD2 %xmm7, %xmm9 ADD2 %xmm6, %xmm11 movaps %xmm5, %xmm6 mulpd %xmm0, %xmm5 movaps 8 * SIZE(AO), %xmm0 mulpd %xmm1, %xmm6 movaps 10 * SIZE(AO), %xmm1 ADD1 %xmm3, %xmm12 movaps 10 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm1, %xmm4 ADD2 %xmm5, %xmm13 ADD2 %xmm6, %xmm15 movaps %xmm7, %xmm6 mulpd %xmm0, %xmm7 mulpd %xmm1, %xmm6 ADD1 %xmm2, %xmm8 movaps 12 * SIZE(BO), %xmm2 ADD1 %xmm4, %xmm10 movaps %xmm3, %xmm4 pshufd $0x4e, %xmm3, %xmm5 mulpd %xmm0, %xmm3 mulpd %xmm1, %xmm4 ADD2 %xmm7, %xmm9 ADD2 %xmm6, %xmm11 movaps %xmm5, %xmm6 mulpd %xmm0, %xmm5 movaps 12 * SIZE(AO), %xmm0 mulpd %xmm1, %xmm6 movaps 14 * SIZE(AO), %xmm1 ADD1 %xmm3, %xmm12 movaps 14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm1, %xmm4 ADD2 %xmm5, %xmm13 ADD2 %xmm6, %xmm15 movaps %xmm7, %xmm6 mulpd %xmm0, %xmm7 mulpd %xmm1, %xmm6 ADD1 %xmm2, %xmm8 movaps 16 * SIZE(BO), %xmm2 ADD1 %xmm4, %xmm10 movaps %xmm3, %xmm4 pshufd $0x4e, %xmm3, %xmm5 mulpd %xmm0, %xmm3 mulpd %xmm1, %xmm4 ADD2 %xmm7, %xmm9 subq $-32 * SIZE, AO ADD2 %xmm6, %xmm11 movaps %xmm5, %xmm6 mulpd %xmm0, %xmm5 movaps -16 * SIZE(AO), %xmm0 mulpd %xmm1, %xmm6 movaps -14 * SIZE(AO), %xmm1 subq $-32 * SIZE, BO subq $1, %rax jg,pt .L12 ALIGN_3.L15:#if defined(LT) || defined(RN) movq KK, %rax#else movq K, %rax subq KK, %rax#endif andq $7, %rax # if (k & 1) je,pt .L18 ALIGN_3.L16: ADD1 %xmm3, %xmm12 movaps -14 * SIZE(BO), %xmm3 ADD1 %xmm4, %xmm14 movaps %xmm2, %xmm4 pshufd $0x4e, %xmm2, %xmm7 mulpd %xmm0, %xmm2 mulpd %xmm1, %xmm4 ADD2 %xmm5, %xmm13 ADD2 %xmm6, %xmm15 movaps %xmm7, %xmm6 mulpd %xmm0, %xmm7 mulpd %xmm1, %xmm6 ADD1 %xmm2, %xmm8 movaps -12 * SIZE(BO), %xmm2 ADD1 %xmm4, %xmm10 movaps %xmm3, %xmm4 pshufd $0x4e, %xmm3, %xmm5 mulpd %xmm0, %xmm3 mulpd %xmm1, %xmm4 ADD2 %xmm7, %xmm9 ADD2 %xmm6, %xmm11 movaps %xmm5, %xmm6 mulpd %xmm0, %xmm5 movaps -12 * SIZE(AO), %xmm0 mulpd %xmm1, %xmm6 movaps -10 * SIZE(AO), %xmm1 addq $4 * SIZE, AO addq $4 * SIZE, BO subq $1, %rax jg,pt .L16 ALIGN_3.L18:#if defined(LN) || defined(RT) movq KK, %rax subq $2, %rax leaq (, %rax, SIZE), %rax movq AORIG, AO leaq (AO, %rax, 4), AO leaq (B, %rax, 4), BO#endif ADD1 %xmm3, %xmm12 pcmpeqb %xmm7, %xmm7 ADD1 %xmm4, %xmm14 psllq $63, %xmm7 ADD2 %xmm5, %xmm13 ADD2 %xmm6, %xmm15#ifndef CONJ pshufd $0x40, %xmm7, %xmm0 shufps $0x04, %xmm7, %xmm7 pxor %xmm0, %xmm8 pxor %xmm0, %xmm10 pxor %xmm0, %xmm12 pxor %xmm0, %xmm14#else#if defined(LN) || defined(LT) pshufd $0x40, %xmm7, %xmm0#else pshufd $0x04, %xmm7, %xmm0#endif shufps $0x40, %xmm7, %xmm7 pxor %xmm0, %xmm9 pxor %xmm0, %xmm11 pxor %xmm0, %xmm13 pxor %xmm0, %xmm15#endif haddpd %xmm9, %xmm8 haddpd %xmm11, %xmm10 haddpd %xmm13, %xmm12 haddpd %xmm15, %xmm14#if defined(LN) || defined(LT) movapd -16 * SIZE(BO), %xmm9 movapd -14 * SIZE(BO), %xmm11 movapd -12 * SIZE(BO), %xmm13 movapd -10 * SIZE(BO), %xmm15 subpd %xmm8, %xmm9 subpd %xmm12, %xmm11 subpd %xmm10, %xmm13 subpd %xmm14, %xmm15#else movapd -16 * SIZE(AO), %xmm9 movapd -14 * SIZE(AO), %xmm13 movapd -12 * SIZE(AO), %xmm11 movapd -10 * SIZE(AO), %xmm15 subpd %xmm8, %xmm9 subpd %xmm12, %xmm11 subpd %xmm10, %xmm13 subpd %xmm14, %xmm15#endif#ifdef LN movddup -10 * SIZE(AO), %xmm0 movddup -9 * SIZE(AO), %xmm1 movddup -12 * SIZE(AO), %xmm2 movddup -11 * SIZE(AO), %xmm3 movddup -16 * SIZE(AO), %xmm4 movddup -15 * SIZE(AO), %xmm5 pshufd $0x4e, %xmm13, %xmm12 pshufd $0x4e, %xmm15, %xmm14 xorpd %xmm7, %xmm12 xorpd %xmm7, %xmm14 mulpd %xmm0, %xmm13 mulpd %xmm1, %xmm12 mulpd %xmm0, %xmm15 mulpd %xmm1, %xmm14 addpd %xmm12, %xmm13 addpd %xmm14, %xmm15 movapd %xmm13, %xmm8 movapd %xmm15, %xmm10 pshufd $0x4e, %xmm13, %xmm12 pshufd $0x4e, %xmm15, %xmm14 xorpd %xmm7, %xmm12 xorpd %xmm7, %xmm14 mulpd %xmm2, %xmm8 mulpd %xmm2, %xmm10 mulpd %xmm3, %xmm12 mulpd %xmm3, %xmm14 subpd %xmm8, %xmm9 subpd %xmm10, %xmm11 subpd %xmm12, %xmm9⌨️ 快捷键说明
复制代码
Ctrl + C
搜索代码
Ctrl + F
全屏模式
F11
切换主题
Ctrl + Shift + D
显示快捷键
?
增大字号
Ctrl + =
减小字号
Ctrl + -