📄 trsm_kernel_ln_4x2_core2.s
字号:
/*********************************************************************//* *//* Optimized BLAS libraries *//* By Kazushige Goto <kgoto@tacc.utexas.edu> *//* *//* Copyright (c) The University of Texas, 2005. All rights reserved. *//* UNIVERSITY EXPRESSLY DISCLAIMS ANY AND ALL WARRANTIES CONCERNING *//* THIS SOFTWARE AND DOCUMENTATION, INCLUDING ANY WARRANTIES OF *//* MERCHANTABILITY, FITNESS FOR ANY PARTICULAR PURPOSE, *//* NON-INFRINGEMENT AND WARRANTIES OF PERFORMANCE, AND ANY WARRANTY *//* THAT MIGHT OTHERWISE ARISE FROM COURSE OF DEALING OR USAGE OF *//* TRADE. NO WARRANTY IS EITHER EXPRESS OR IMPLIED WITH RESPECT TO *//* THE USE OF THE SOFTWARE OR DOCUMENTATION. *//* Under no circumstances shall University be liable for incidental, *//* special, indirect, direct or consequential damages or loss of *//* profits, interruption of business, or related expenses which may *//* arise from use of Software or Documentation, including but not *//* limited to those resulting from defects in Software and/or *//* Documentation, or loss or inaccuracy of data of any kind. *//*********************************************************************/#define ASSEMBLER#include "common.h"#if !defined(HAVE_SSE2) || !defined(HAVE_MMX)#error You have to check your configuration.#endif#define STACK 16#define ARGS 0 #define STACK_M 4 + STACK + ARGS(%esi)#define STACK_N 8 + STACK + ARGS(%esi)#define STACK_K 12 + STACK + ARGS(%esi)#define STACK_ALPHA 16 + STACK + ARGS(%esi)#define STACK_A 24 + STACK + ARGS(%esi)#define STACK_B 28 + STACK + ARGS(%esi)#define STACK_C 32 + STACK + ARGS(%esi)#define STACK_LDC 36 + STACK + ARGS(%esi)#define STACK_OFFT 40 + STACK + ARGS(%esi) #define K 16(%esp)#define N 20(%esp)#define M 24(%esp)#define A 28(%esp)#define C 32(%esp)#define J 36(%esp)#define OLD_STACK 40(%esp)#define OFFSET 44(%esp)#define KK 48(%esp)#define KKK 52(%esp)#define AORIG 56(%esp)#define BORIG 60(%esp)#define BUFFER 128(%esp)#define STACK_ALIGN 4096#define STACK_OFFSET 1024#define LOCAL_BUFFER_SIZE GEMM_Q * GEMM_UNROLL_N * COMPSIZE * 16#define B %edi#define AA %edx#define BB %ecx#define LDC %ebp#define CO1 %esi PROLOGUE pushl %ebp pushl %edi pushl %esi pushl %ebx PROFCODE EMMS movl %esp, %esi # save old stack subl $128 + LOCAL_BUFFER_SIZE + STACK_OFFSET, %esp andl $-STACK_ALIGN, %esp addl $STACK_OFFSET, %esp#ifdef WINDOWS_ABI#if GEMM_Q > 128 movl $0, 4096 * 1(%esp)#endif movl $0, 4096 * 0(%esp)#endif movd STACK_M, %mm0 movl STACK_N, %eax movd STACK_K, %mm1 movd STACK_A, %mm2 movl STACK_B, B movd STACK_C, %mm3 movl STACK_LDC, LDC movd STACK_OFFT, %mm4 movd %mm1, K movl %eax, N movd %mm0, M movd %mm2, A movd %mm3, C movl %esi, OLD_STACK movd %mm4, OFFSET movd %mm4, KK subl $-16 * SIZE, A subl $-16 * SIZE, B sall $BASE_SHIFT, LDC#ifdef LN movl M, %eax leal (, %eax, SIZE), %eax addl %eax, C imull K, %eax addl %eax, A#endif#ifdef RT movl N, %eax leal (, %eax, SIZE), %eax imull K, %eax addl %eax, B movl N, %eax imull LDC, %eax addl %eax, C#endif#ifdef RN negl KK#endif #ifdef RT movl N, %eax subl OFFSET, %eax movl %eax, KK#endif movl N, %eax sarl $1, %eax movl %eax, J jle .L100 ALIGN_2 .L01:/* Copying to Sub Buffer */#ifdef LN movl OFFSET, %eax addl M, %eax movl %eax, KK#endif leal 16 * SIZE + BUFFER, BB#ifdef RT movl K, %eax sall $1 + BASE_SHIFT, %eax subl %eax, B#endif#if defined(LN) || defined(RT) movl KK, %eax movl B, BORIG leal (, %eax, SIZE), %eax leal (B, %eax, 2), B leal (BB, %eax, 4), BB#endif #ifdef LT movl OFFSET, %eax movl %eax, KK#endif#if defined(LT) || defined(RN) movl KK, %eax#else movl K, %eax subl KK, %eax#endif sarl $2, %eax jle .L03 ALIGN_2 .L02: movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movddup -14 * SIZE(B), %xmm2 movddup -13 * SIZE(B), %xmm3 movddup -12 * SIZE(B), %xmm4 movddup -11 * SIZE(B), %xmm5 movddup -10 * SIZE(B), %xmm6 movddup -9 * SIZE(B), %xmm7 movapd %xmm0, -16 * SIZE(BB) movapd %xmm1, -14 * SIZE(BB) movapd %xmm2, -12 * SIZE(BB) movapd %xmm3, -10 * SIZE(BB) movapd %xmm4, -8 * SIZE(BB) movapd %xmm5, -6 * SIZE(BB) movapd %xmm6, -4 * SIZE(BB) movapd %xmm7, -2 * SIZE(BB) addl $ 8 * SIZE, B addl $16 * SIZE, %ecx decl %eax jne .L02 ALIGN_2.L03:#if defined(LT) || defined(RN) movl KK, %eax#else movl K, %eax subl KK, %eax#endif andl $3, %eax BRANCH jle .L05 ALIGN_4.L04: movddup -16 * SIZE(B), %xmm0 movddup -15 * SIZE(B), %xmm1 movapd %xmm0, -16 * SIZE(BB) movapd %xmm1, -14 * SIZE(BB) addl $2 * SIZE, B addl $4 * SIZE, %ecx decl %eax jne .L04 ALIGN_4 .L05:#if defined(LT) || defined(RN) movl A, AA#else movl A, %eax movl %eax, AORIG#endif leal (, LDC, 2), %eax#ifdef RT subl %eax, C#endif movl C, CO1 # coffset = c#ifndef RT addl %eax, C#endif movl M, %ebx testl $1, %ebx jle .L30#ifdef LN movl K, %eax sall $0 + BASE_SHIFT, %eax subl %eax, AORIG#endif#if defined(LN) || defined(RT) movl KK, %eax movl AORIG, AA leal (, %eax, SIZE), %eax leal (AA, %eax, 1), AA#endif leal 16 * SIZE + BUFFER, BB#if defined(LN) || defined(RT) movl KK, %eax sall $1 + BASE_SHIFT, %eax leal (BB, %eax, 2), BB#endif movsd -16 * SIZE(AA), %xmm0 pxor %xmm4, %xmm4 movsd -16 * SIZE(BB), %xmm1 pxor %xmm5, %xmm5 movsd -12 * SIZE(AA), %xmm2 pxor %xmm6, %xmm6 movsd -8 * SIZE(BB), %xmm3 pxor %xmm7, %xmm7#if defined(LT) || defined(RN) movl KK, %eax#else movl K, %eax subl KK, %eax#endif sarl $3, %eax je .L52.L51: mulsd %xmm0, %xmm1 mulsd -14 * SIZE(BB), %xmm0 addsd %xmm1, %xmm4 movsd -12 * SIZE(BB), %xmm1 addsd %xmm0, %xmm5 movsd -15 * SIZE(AA), %xmm0 mulsd %xmm0, %xmm1 mulsd -10 * SIZE(BB), %xmm0 addsd %xmm1, %xmm6 movsd 0 * SIZE(BB), %xmm1 addsd %xmm0, %xmm7 movsd -14 * SIZE(AA), %xmm0 mulsd %xmm0, %xmm3 mulsd -6 * SIZE(BB), %xmm0 addsd %xmm3, %xmm4 movsd -4 * SIZE(BB), %xmm3 addsd %xmm0, %xmm5 movsd -13 * SIZE(AA), %xmm0 mulsd %xmm0, %xmm3 mulsd -2 * SIZE(BB), %xmm0 addsd %xmm3, %xmm6 movsd 8 * SIZE(BB), %xmm3 addsd %xmm0, %xmm7 movsd -8 * SIZE(AA), %xmm0 mulsd %xmm2, %xmm1 mulsd 2 * SIZE(BB), %xmm2 addsd %xmm1, %xmm4 movsd 4 * SIZE(BB), %xmm1 addsd %xmm2, %xmm5 movsd -11 * SIZE(AA), %xmm2 mulsd %xmm2, %xmm1 mulsd 6 * SIZE(BB), %xmm2 addsd %xmm1, %xmm6 movsd 16 * SIZE(BB), %xmm1 addsd %xmm2, %xmm7 movsd -10 * SIZE(AA), %xmm2 mulsd %xmm2, %xmm3 mulsd 10 * SIZE(BB), %xmm2 addsd %xmm3, %xmm4 movsd 12 * SIZE(BB), %xmm3 addsd %xmm2, %xmm5 movsd -9 * SIZE(AA), %xmm2 mulsd %xmm2, %xmm3 mulsd 14 * SIZE(BB), %xmm2 addsd %xmm3, %xmm6 movsd 24 * SIZE(BB), %xmm3 addsd %xmm2, %xmm7 movsd -4 * SIZE(AA), %xmm2 subl $-8 * SIZE, AA addl $32 * SIZE, BB decl %eax jne .L51.L52:#if defined(LT) || defined(RN) movl KK, %eax#else movl K, %eax subl KK, %eax#endif andl $7, %eax # if (k & 1) BRANCH je .L54.L53: mulsd %xmm0, %xmm1 mulsd -14 * SIZE(BB), %xmm0 addsd %xmm1, %xmm4 movsd -12 * SIZE(BB), %xmm1 addsd %xmm0, %xmm5 movsd -15 * SIZE(AA), %xmm0 addl $1 * SIZE, AA addl $4 * SIZE, BB decl %eax BRANCH jg .L53 ALIGN_4.L54: addsd %xmm6, %xmm4 addsd %xmm7, %xmm5#if defined(LN) || defined(RT) movl KK, %eax#ifdef LN subl $1, %eax#else subl $2, %eax#endif movl AORIG, AA movl BORIG, B leal 16 * SIZE + BUFFER, BB leal (, %eax, SIZE), %eax leal (AA, %eax, 1), AA leal (B, %eax, 2), B leal (BB, %eax, 4), BB#endif#if defined(LN) || defined(LT) movsd -16 * SIZE(B), %xmm0 movsd -15 * SIZE(B), %xmm1#else movsd -16 * SIZE(AA), %xmm0 movsd -15 * SIZE(AA), %xmm1#endif subsd %xmm4, %xmm0 subsd %xmm5, %xmm1#if defined(LN) || defined(LT) movsd -16 * SIZE(AA), %xmm2 mulsd %xmm2, %xmm0 mulsd %xmm2, %xmm1#endif#ifdef RN mulsd -16 * SIZE(B), %xmm0 movsd -15 * SIZE(B), %xmm4 mulsd %xmm0, %xmm4 subsd %xmm4, %xmm1 mulsd -13 * SIZE(B), %xmm1#endif#ifdef RT mulsd -13 * SIZE(B), %xmm1 movsd -14 * SIZE(B), %xmm4 mulsd %xmm1, %xmm4 subsd %xmm4, %xmm0 mulsd -16 * SIZE(B), %xmm0#endif#ifdef LN subl $1 * SIZE, CO1#endif movsd %xmm0, 0 * SIZE(CO1) movsd %xmm1, 0 * SIZE(CO1, LDC)#ifndef LN addl $1 * SIZE, CO1#endif#if defined(LN) || defined(LT) movsd %xmm0, -16 * SIZE(B) movsd %xmm1, -15 * SIZE(B) movsd %xmm0, -16 * SIZE(BB) movsd %xmm0, -15 * SIZE(BB) movsd %xmm1, -14 * SIZE(BB) movsd %xmm1, -13 * SIZE(BB)#else movsd %xmm0, -16 * SIZE(AA) movsd %xmm1, -15 * SIZE(AA)#endif#if defined(LT) || defined(RN) movl K, %eax subl KK, %eax leal (,%eax, SIZE), %eax leal (AA, %eax, 1), AA#ifdef LT addl $2 * SIZE, B#endif#endif#ifdef LN subl $1, KK movl BORIG, B#endif#ifdef LT addl $1, KK#endif#ifdef RT movl K, %eax movl BORIG, B sall $0 + BASE_SHIFT, %eax addl %eax, AORIG#endif ALIGN_2.L30: movl M, %ebx testl $2, %ebx jle .L50#ifdef LN movl K, %eax sall $1 + BASE_SHIFT, %eax subl %eax, AORIG#endif#if defined(LN) || defined(RT) movl KK, %eax movl AORIG, AA leal (, %eax, SIZE), %eax leal (AA, %eax, 2), AA#endif leal 16 * SIZE + BUFFER, BB#if defined(LN) || defined(RT) movl KK, %eax sall $1 + BASE_SHIFT, %eax leal (BB, %eax, 2), BB#endif movapd -16 * SIZE(AA), %xmm0 pxor %xmm4, %xmm4 movapd -16 * SIZE(BB), %xmm1 pxor %xmm5, %xmm5 movapd -8 * SIZE(AA), %xmm2 pxor %xmm6, %xmm6 movapd -8 * SIZE(BB), %xmm3 pxor %xmm7, %xmm7#if defined(LT) || defined(RN) movl KK, %eax#else movl K, %eax subl KK, %eax#endif sarl $3, %eax je .L32.L31: mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BB), %xmm0 addpd %xmm1, %xmm4 movapd -12 * SIZE(BB), %xmm1 addpd %xmm0, %xmm5 movapd -14 * SIZE(AA), %xmm0 mulpd %xmm0, %xmm1 mulpd -10 * SIZE(BB), %xmm0 addpd %xmm1, %xmm6 movapd 0 * SIZE(BB), %xmm1 addpd %xmm0, %xmm7 movapd -12 * SIZE(AA), %xmm0 mulpd %xmm0, %xmm3 mulpd -6 * SIZE(BB), %xmm0 addpd %xmm3, %xmm4 movapd -4 * SIZE(BB), %xmm3 addpd %xmm0, %xmm5 movapd -10 * SIZE(AA), %xmm0 mulpd %xmm0, %xmm3 mulpd -2 * SIZE(BB), %xmm0 addpd %xmm3, %xmm6 movapd 8 * SIZE(BB), %xmm3 addpd %xmm0, %xmm7 movapd 0 * SIZE(AA), %xmm0 mulpd %xmm2, %xmm1 mulpd 2 * SIZE(BB), %xmm2 addpd %xmm1, %xmm4 movapd 4 * SIZE(BB), %xmm1 addpd %xmm2, %xmm5 movapd -6 * SIZE(AA), %xmm2 mulpd %xmm2, %xmm1 mulpd 6 * SIZE(BB), %xmm2 addpd %xmm1, %xmm6 movapd 16 * SIZE(BB), %xmm1 addpd %xmm2, %xmm7 movapd -4 * SIZE(AA), %xmm2 mulpd %xmm2, %xmm3 mulpd 10 * SIZE(BB), %xmm2 addpd %xmm3, %xmm4 movapd 12 * SIZE(BB), %xmm3 addpd %xmm2, %xmm5 movapd -2 * SIZE(AA), %xmm2 mulpd %xmm2, %xmm3 mulpd 14 * SIZE(BB), %xmm2 addpd %xmm3, %xmm6 movapd 24 * SIZE(BB), %xmm3 addpd %xmm2, %xmm7 movapd 8 * SIZE(AA), %xmm2 subl $-16 * SIZE, AA addl $ 32 * SIZE, BB decl %eax jne .L31.L32:#if defined(LT) || defined(RN) movl KK, %eax#else movl K, %eax subl KK, %eax#endif andl $7, %eax # if (k & 1) BRANCH je .L34.L33: mulpd %xmm0, %xmm1 mulpd -14 * SIZE(BB), %xmm0 addpd %xmm1, %xmm4 movapd -12 * SIZE(BB), %xmm1 addpd %xmm0, %xmm5 movapd -14 * SIZE(AA), %xmm0 addl $2 * SIZE, AA addl $4 * SIZE, BB decl %eax jg .L33 ALIGN_4.L34: addpd %xmm6, %xmm4 addpd %xmm7, %xmm5#if defined(LN) || defined(RT) movl KK, %eax#ifdef LN subl $2, %eax#else subl $2, %eax#endif movl AORIG, AA movl BORIG, B leal 16 * SIZE + BUFFER, BB leal (, %eax, SIZE), %eax leal (AA, %eax, 2), AA leal (B, %eax, 2), B leal (BB, %eax, 4), BB#endif#if defined(LN) || defined(LT) movapd %xmm4, %xmm0 unpcklpd %xmm5, %xmm4 unpckhpd %xmm5, %xmm0 movapd -16 * SIZE(B), %xmm2 movapd -14 * SIZE(B), %xmm3 subpd %xmm4, %xmm2 subpd %xmm0, %xmm3#else movapd -16 * SIZE(AA), %xmm0 movapd -14 * SIZE(AA), %xmm1 subpd %xmm4, %xmm0 subpd %xmm5, %xmm1#endif#ifdef LN movddup -13 * SIZE(AA), %xmm0 mulpd %xmm0, %xmm3 movddup -14 * SIZE(AA), %xmm0 mulpd %xmm3, %xmm0 subpd %xmm0, %xmm2 movddup -16 * SIZE(AA), %xmm0 mulpd %xmm0, %xmm2#endif#ifdef LT movddup -16 * SIZE(AA), %xmm0 mulpd %xmm0, %xmm2 movddup -15 * SIZE(AA), %xmm0 mulpd %xmm2, %xmm0 subpd %xmm0, %xmm3 movddup -13 * SIZE(AA), %xmm0 mulpd %xmm0, %xmm3#endif#ifdef RN movddup -16 * SIZE(B), %xmm4 mulpd %xmm4, %xmm0 movddup -15 * SIZE(B), %xmm4 mulpd %xmm0, %xmm4 subpd %xmm4, %xmm1 movddup -13 * SIZE(B), %xmm4 mulpd %xmm4, %xmm1#endif#ifdef RT movddup -13 * SIZE(B), %xmm4 mulpd %xmm4, %xmm1 movddup -14 * SIZE(B), %xmm4 mulpd %xmm1, %xmm4 subpd %xmm4, %xmm0 movddup -16 * SIZE(B), %xmm4 mulpd %xmm4, %xmm0#endif#ifdef LN subl $2 * SIZE, CO1#endif#if defined(LN) || defined(LT) movsd %xmm2, 0 * SIZE(CO1) movsd %xmm3, 1 * SIZE(CO1) movhpd %xmm2, 0 * SIZE(CO1, LDC)⌨️ 快捷键说明
复制代码
Ctrl + C
搜索代码
Ctrl + F
全屏模式
F11
切换主题
Ctrl + Shift + D
显示快捷键
?
增大字号
Ctrl + =
减小字号
Ctrl + -