📄 gemm_kernel_8x1_sse2.s

📁 Optimized GotoBLAS libraries
💻 S
📖 第 1 页 / 共 2 页
字号:
12 下一页
/*********************************************************************//*                                                                   *//*             Optimized BLAS libraries                              *//*                     By Kazushige Goto <kgoto@tacc.utexas.edu>     *//*                                                                   *//* Copyright (c) The University of Texas, 2005. All rights reserved. *//* UNIVERSITY EXPRESSLY DISCLAIMS ANY AND ALL WARRANTIES CONCERNING  *//* THIS SOFTWARE AND DOCUMENTATION, INCLUDING ANY WARRANTIES OF      *//* MERCHANTABILITY, FITNESS FOR ANY PARTICULAR PURPOSE,              *//* NON-INFRINGEMENT AND WARRANTIES OF PERFORMANCE, AND ANY WARRANTY  *//* THAT MIGHT OTHERWISE ARISE FROM COURSE OF DEALING OR USAGE OF     *//* TRADE. NO WARRANTY IS EITHER EXPRESS OR IMPLIED WITH RESPECT TO   *//* THE USE OF THE SOFTWARE OR DOCUMENTATION.                         *//* Under no circumstances shall University be liable for incidental, *//* special, indirect, direct or consequential damages or loss of     *//* profits, interruption of business, or related expenses which may  *//* arise from use of Software or Documentation, including but not    *//* limited to those resulting from defects in Software and/or        *//* Documentation, or loss or inaccuracy of data of any kind.         *//*********************************************************************/#define ASSEMBLER#include "common.h"#if !defined(HAVE_SSE2) || !defined(HAVE_MMX)#error  You have to check your configuration.#endif#define STACK	16#define ARGS	 0	#define STACK_M	 4 + STACK + ARGS(%esi)#define STACK_N	 8 + STACK + ARGS(%esi)#define STACK_K	12 + STACK + ARGS(%esi)#define STACK_ALPHA	16 + STACK + ARGS(%esi)#define STACK_A	24 + STACK + ARGS(%esi)#define STACK_B	28 + STACK + ARGS(%esi)#define STACK_C	32 + STACK + ARGS(%esi)#define STACK_LDC	36 + STACK + ARGS(%esi)#define ALPHA	 0(%esp)#define K	16(%esp)#define N	20(%esp)#define M	24(%esp)#define A	28(%esp)#define C	36(%esp)#define J	44(%esp)#define OLD_STACK 48(%esp)#define BUFFER 128(%esp)#define B	%edi#define LDC	%ebp#define STACK_ALIGN	4096#define STACK_OFFSET	1024#define LOCAL_BUFFER_SIZE  GEMM_Q * GEMM_UNROLL_N * COMPSIZE * 16#define AA	%edx#define BB	%ecx#define KERNELMACRO(address) \	mulpd	%xmm0, %xmm2; \	mulpd	 2 * SIZE + (address) * SIZE(BB), %xmm0; \	addpd	%xmm2, %xmm4; \	movapd	 0 * SIZE + (address) * SIZE(BB), %xmm2; \	addpd	%xmm0, %xmm5; \	movapd	 2 * SIZE + (address) * SIZE(AA), %xmm0; \	mulpd	%xmm0, %xmm2; \	mulpd	 2 * SIZE + (address) * SIZE(BB), %xmm0; \	addpd	%xmm2, %xmm6; \	movapd	 4 * SIZE + (address) * SIZE(BB), %xmm2; \	addpd	%xmm0, %xmm7; \	movapd	 4 * SIZE + (address) * SIZE(AA), %xmm0; \	mulpd	%xmm0, %xmm2; \	mulpd	 6 * SIZE + (address) * SIZE(BB), %xmm0; \	addpd	%xmm2, %xmm4; \	movapd	 4 * SIZE + (address) * SIZE(BB), %xmm2; \	addpd	%xmm0, %xmm5; \	movapd	 6 * SIZE + (address) * SIZE(AA), %xmm0; \	mulpd	%xmm0, %xmm2; \	mulpd	 6 * SIZE + (address) * SIZE(BB), %xmm0; \	addpd	%xmm2, %xmm6; \	movapd	16 * SIZE + (address) * SIZE(BB), %xmm2; \	addpd	%xmm0, %xmm7; \	movapd	16 * SIZE + (address) * SIZE(AA), %xmm0; \	mulpd	%xmm1, %xmm3; \	mulpd	10 * SIZE + (address) * SIZE(BB), %xmm1; \	addpd	%xmm3, %xmm4; \	movapd	 8 * SIZE + (address) * SIZE(BB), %xmm3; \	addpd	%xmm1, %xmm5; \	movapd	10 * SIZE + (address) * SIZE(AA), %xmm1; \	mulpd	%xmm1, %xmm3; \	mulpd	10 * SIZE + (address) * SIZE(BB), %xmm1; \	addpd	%xmm3, %xmm6; \	movapd	12 * SIZE + (address) * SIZE(BB), %xmm3; \	addpd	%xmm1, %xmm7; \	movapd	12 * SIZE + (address) * SIZE(AA), %xmm1; \	mulpd	%xmm1, %xmm3; \	mulpd	14 * SIZE + (address) * SIZE(BB), %xmm1; \	addpd	%xmm3, %xmm4; \	movapd	12 * SIZE + (address) * SIZE(BB), %xmm3; \	addpd	%xmm1, %xmm5; \	movapd	14 * SIZE + (address) * SIZE(AA), %xmm1; \	mulpd	%xmm1, %xmm3; \	mulpd	14 * SIZE + (address) * SIZE(BB), %xmm1; \	addpd	%xmm3, %xmm6; \	movapd	24 * SIZE + (address) * SIZE(BB), %xmm3; \	addpd	%xmm1, %xmm7; \	movapd	24 * SIZE + (address) * SIZE(AA), %xmm1; \	mulpd	%xmm0, %xmm2; \	mulpd	18 * SIZE + (address) * SIZE(BB), %xmm0; \	addpd	%xmm2, %xmm4; \	movapd	16 * SIZE + (address) * SIZE(BB), %xmm2; \	addpd	%xmm0, %xmm5; \	movapd	18 * SIZE + (address) * SIZE(AA), %xmm0; \	mulpd	%xmm0, %xmm2; \	mulpd	18 * SIZE + (address) * SIZE(BB), %xmm0; \	addpd	%xmm2, %xmm6; \	movapd	20 * SIZE + (address) * SIZE(BB), %xmm2; \	addpd	%xmm0, %xmm7; \	movapd	20 * SIZE + (address) * SIZE(AA), %xmm0; \	mulpd	%xmm0, %xmm2; \	mulpd	22 * SIZE + (address) * SIZE(BB), %xmm0; \	addpd	%xmm2, %xmm4; \	movapd	20 * SIZE + (address) * SIZE(BB), %xmm2; \	addpd	%xmm0, %xmm5; \	movapd	22 * SIZE + (address) * SIZE(AA), %xmm0; \	mulpd	%xmm0, %xmm2; \	mulpd	22 * SIZE + (address) * SIZE(BB), %xmm0; \	addpd	%xmm2, %xmm6; \	movapd	32 * SIZE + (address) * SIZE(BB), %xmm2; \	addpd	%xmm0, %xmm7; \	movapd	32 * SIZE + (address) * SIZE(AA), %xmm0; \	mulpd	%xmm1, %xmm3; \	mulpd	26 * SIZE + (address) * SIZE(BB), %xmm1; \	addpd	%xmm3, %xmm4; \	movapd	24 * SIZE + (address) * SIZE(BB), %xmm3; \	addpd	%xmm1, %xmm5; \	movapd	26 * SIZE + (address) * SIZE(AA), %xmm1; \	mulpd	%xmm1, %xmm3; \	mulpd	26 * SIZE + (address) * SIZE(BB), %xmm1; \	addpd	%xmm3, %xmm6; \	movapd	28 * SIZE + (address) * SIZE(BB), %xmm3; \	addpd	%xmm1, %xmm7; \	movapd	28 * SIZE + (address) * SIZE(AA), %xmm1; \	mulpd	%xmm1, %xmm3; \	mulpd	30 * SIZE + (address) * SIZE(BB), %xmm1; \	addpd	%xmm3, %xmm4; \	movapd	28 * SIZE + (address) * SIZE(BB), %xmm3; \	addpd	%xmm1, %xmm5; \	movapd	30 * SIZE + (address) * SIZE(AA), %xmm1; \	mulpd	%xmm1, %xmm3; \	mulpd	30 * SIZE + (address) * SIZE(BB), %xmm1; \	addpd	%xmm3, %xmm6; \	movapd	40 * SIZE + (address) * SIZE(BB), %xmm3; \	addpd	%xmm1, %xmm7; \	movapd	40 * SIZE + (address) * SIZE(AA), %xmm1	PROLOGUE	pushl	%ebp	pushl	%edi	pushl	%esi	pushl	%ebx	PROFCODE	EMMS	movl	%esp, %esi	# save old stack	subl	$128 + LOCAL_BUFFER_SIZE + STACK_OFFSET, %esp	andl	$-STACK_ALIGN, %esp	addl	$STACK_OFFSET, %esp#ifdef WINDOWS_ABI#if LOCAL_BUFFER_SIZE > 12288	movl	$0,  4096 * 3(%esp)#endif#if LOCAL_BUFFER_SIZE >  8192	movl	$0,  4096 * 2(%esp)#endif#if LOCAL_BUFFER_SIZE >  4096	movl	$0,  4096 * 1(%esp)#endif	movl	$0,  4096 * 0(%esp)#endif	movd	STACK_M, %mm0	movl	STACK_N, %eax	movd	STACK_K, %mm1	movd	STACK_A, %mm2	movq	STACK_ALPHA,  %mm7	movl	STACK_B, B	movd	STACK_C, %mm3	movl	STACK_LDC, LDC	movq	%mm7, 0 * SIZE + ALPHA	movq	%mm7, 1 * SIZE + ALPHA	movd	%mm1, K	movl	%eax, N	movd	%mm0, M	movd	%mm2, A	movd	%mm3, C	movl	%esi, OLD_STACK	leal	(, LDC, SIZE), LDC	test	%eax, %eax	movl	%eax, J	jle	.L999	ALIGN_2	.L01:/* Copying to Sub Buffer */	movl	K, %eax	leal	BUFFER, %ecx	sarl	$3, %eax	jle	.L03	ALIGN_4	.L02:	prefetchnta	 96 * SIZE(B)	movsd	 0 * SIZE(B), %xmm0	movsd	 1 * SIZE(B), %xmm1	movsd	 2 * SIZE(B), %xmm2	movsd	 3 * SIZE(B), %xmm3	movsd	 4 * SIZE(B), %xmm4	movsd	 5 * SIZE(B), %xmm5	movsd	 6 * SIZE(B), %xmm6	movsd	 7 * SIZE(B), %xmm7	unpcklpd  %xmm0, %xmm0	unpcklpd  %xmm1, %xmm1	unpcklpd  %xmm2, %xmm2	unpcklpd  %xmm3, %xmm3	unpcklpd  %xmm4, %xmm4	unpcklpd  %xmm5, %xmm5	unpcklpd  %xmm6, %xmm6	unpcklpd  %xmm7, %xmm7	movapd	%xmm0,  0 * SIZE(%ecx)	movapd	%xmm1,  2 * SIZE(%ecx)	movapd	%xmm2,  4 * SIZE(%ecx)	movapd	%xmm3,  6 * SIZE(%ecx)	movapd	%xmm4,  8 * SIZE(%ecx)	movapd	%xmm5, 10 * SIZE(%ecx)	movapd	%xmm6, 12 * SIZE(%ecx)	movapd	%xmm7, 14 * SIZE(%ecx)	addl	$ 8 * SIZE, B	addl	$16 * SIZE, %ecx	decl	%eax	BRANCH	jne	.L02	ALIGN_2.L03:	movl	K, %eax	andl	$7, %eax	BRANCH	jle	.L05	ALIGN_2.L04:	movsd	 0 * SIZE(B), %xmm0	unpcklpd  %xmm0, %xmm0	movapd	%xmm0,  0 * SIZE(%ecx)	addl	$1 * SIZE, B	addl	$2 * SIZE, %ecx	decl	%eax	jne	.L04	ALIGN_4	.L05:	movl	C, %esi		# coffset = c	movl	A, %edx		# aoffset = a	movl	M,  %ebx	sarl	$3, %ebx	# i = (m >> 2)	jle	.L20	ALIGN_4.L10:	leal	BUFFER, %ecx	# boffset1 = boffset // different point	movl	K, %eax	movapd	 0 * SIZE + BUFFER, %xmm2	movapd	 0 * SIZE(%edx), %xmm0	movapd	 8 * SIZE + BUFFER, %xmm3	movapd	 8 * SIZE(%edx), %xmm1	pxor	%xmm4, %xmm4	pxor	%xmm5, %xmm5	pxor	%xmm6, %xmm6	pxor	%xmm7, %xmm7#if 0	andl	$-8, %eax	leal	(, %eax, 8), %eax	je	.L12		KERNELMACRO(32 *  0)		# 0	cmpl	$64 *  1, %eax	jle	.L11	KERNELMACRO(32 *  1)		# 1	cmpl	$64 *  2, %eax	jle	.L11	KERNELMACRO(32 *  2)		# 2	cmpl	$64 *  3, %eax	jle	.L11	KERNELMACRO(32 *  3)		# 3	cmpl	$64 *  4, %eax	jle	.L11	KERNELMACRO(32 *  4)		# 4	cmpl	$64 *  5, %eax	jle	.L11	KERNELMACRO(32 *  5)		# 5	cmpl	$64 *  6, %eax	jle	.L11	KERNELMACRO(32 *  6)		# 6	cmpl	$64 *  7, %eax	jle	.L11	KERNELMACRO(32 *  7)		# 7	cmpl	$64 *  8, %eax	jle	.L11	KERNELMACRO(32 *  8)		# 8	cmpl	$64 *  9, %eax	jle	.L11	KERNELMACRO(32 *  9)		# 9	cmpl	$64 * 10, %eax	jle	.L11	KERNELMACRO(32 * 10)		# 10	cmpl	$64 * 11, %eax	jle	.L11	KERNELMACRO(32 * 11)		# 11	cmpl	$64 * 12, %eax	jle	.L11	KERNELMACRO(32 * 12)		# 12	cmpl	$64 * 13, %eax	jle	.L11	KERNELMACRO(32 * 13)		# 13	cmpl	$64 * 14, %eax	jle	.L11	KERNELMACRO(32 * 14)		# 14	cmpl	$64 * 15, %eax	jle	.L11	movq	1 * SIZE(%esi), %mm0	movq	1 * SIZE(%esi, LDC), %mm1	KERNELMACRO(32 * 15)		# 15.L11:	leal	(%edx, %eax, 4), %edx	leal	(%ecx, %eax, 4), %ecx#else	movapd	 0 * SIZE(BB), %xmm0	movapd	 8 * SIZE(BB), %xmm2	movapd	 0 * SIZE(AA), %xmm1	movapd	 8 * SIZE(AA), %xmm3	prefetchnta 8 * SIZE(%esi)	sarl	$3, %eax	je	.L12#define PRE 40.L11:		mulpd	%xmm0, %xmm1	movd	(PRE + 0) * SIZE(AA), %mm0	addpd	%xmm1, %xmm4	movapd	 2 * SIZE(AA), %xmm1	mulpd	%xmm0, %xmm1	addpd	%xmm1, %xmm5	movapd	 4 * SIZE(AA), %xmm1	mulpd	%xmm0, %xmm1	mulpd	 6 * SIZE(AA), %xmm0	addpd	%xmm1, %xmm6	movapd	16 * SIZE(AA), %xmm1	movd	(PRE + 8) * SIZE(AA), %mm0	addpd	%xmm0, %xmm7	movapd	 2 * SIZE(BB), %xmm0	mulpd	%xmm0, %xmm3	addpd	%xmm3, %xmm4	movapd	10 * SIZE(AA), %xmm3	mulpd	%xmm0, %xmm3	addpd	%xmm3, %xmm5	movapd	12 * SIZE(AA), %xmm3	mulpd	%xmm0, %xmm3	mulpd	14 * SIZE(AA), %xmm0	addpd	%xmm3, %xmm6	movapd	24 * SIZE(AA), %xmm3	movd	(PRE + 16) * SIZE(AA), %mm0	addpd	%xmm0, %xmm7	movapd	 4 * SIZE(BB), %xmm0	mulpd	%xmm0, %xmm1	addpd	%xmm1, %xmm4	movapd	18 * SIZE(AA), %xmm1	mulpd	%xmm0, %xmm1	addpd	%xmm1, %xmm5	movapd	20 * SIZE(AA), %xmm1	mulpd	%xmm0, %xmm1	mulpd	22 * SIZE(AA), %xmm0	addpd	%xmm1, %xmm6	movapd	32 * SIZE(AA), %xmm1	movd	(PRE + 24) * SIZE(AA), %mm0	addpd	%xmm0, %xmm7	movapd	 6 * SIZE(BB), %xmm0	mulpd	%xmm0, %xmm3	addpd	%xmm3, %xmm4	movapd	26 * SIZE(AA), %xmm3	mulpd	%xmm0, %xmm3	addpd	%xmm3, %xmm5	movapd	28 * SIZE(AA), %xmm3	mulpd	%xmm0, %xmm3	mulpd	30 * SIZE(AA), %xmm0	addpd	%xmm3, %xmm6	movapd	40 * SIZE(AA), %xmm3	movd	(PRE + 32) * SIZE(AA), %mm0	addpd	%xmm0, %xmm7	movapd	16 * SIZE(BB), %xmm0	mulpd	%xmm2, %xmm1	addpd	%xmm1, %xmm4	movapd	34 * SIZE(AA), %xmm1	mulpd	%xmm2, %xmm1	addpd	%xmm1, %xmm5	movapd	36 * SIZE(AA), %xmm1	mulpd	%xmm2, %xmm1	mulpd	38 * SIZE(AA), %xmm2	addpd	%xmm1, %xmm6	movapd	48 * SIZE(AA), %xmm1	movd	(PRE + 40) * SIZE(AA), %mm0	addpd	%xmm2, %xmm7	movapd	10 * SIZE(BB), %xmm2	mulpd	%xmm2, %xmm3
12 下一页
⌨️ 快捷键说明

复制代码 Ctrl + C
搜索代码 Ctrl + F
全屏模式 F11
切换主题 Ctrl + Shift + D
显示快捷键 ?
增大字号 Ctrl + =
减小字号 Ctrl + -