ztrsm_kernel_ln.s

Optimized GotoBLAS libraries

/*********************************************************************/
/*                                                                   */
/*             Optimized BLAS libraries                              */
/*                     By Kazushige Goto <[e-mail]>     */
/*                                                                   */
/* Copyright (c) The University of Texas, 2005. All rights reserved. */
/* UNIVERSITY EXPRESSLY DISCLAIMS ANY AND ALL WARRANTIES CONCERNING  */
/* THIS SOFTWARE AND DOCUMENTATION, INCLUDING ANY WARRANTIES OF      */
/* MERCHANTABILITY, FITNESS FOR ANY PARTICULAR PURPOSE,              */
/* NON-INFRINGEMENT AND WARRANTIES OF PERFORMANCE, AND ANY WARRANTY  */
/* THAT MIGHT OTHERWISE ARISE FROM COURSE OF DEALING OR USAGE OF     */
/* TRADE. NO WARRANTY IS EITHER EXPRESS OR IMPLIED WITH RESPECT TO   */
/* THE USE OF THE SOFTWARE OR DOCUMENTATION.                         */
/* Under no circumstances shall University be liable for incidental, */
/* special, indirect, direct or consequential damages or loss of     */
/* profits, interruption of business, or related expenses which may  */
/* arise from use of Software or Documentation, including but not    */
/* limited to those resulting from defects in Software and/or        */
/* Documentation, or loss or inaccuracy of data of any kind.         */
/*********************************************************************/

#define ASSEMBLER
#include "common.h"

#ifdef DOUBLE
#define PREFETCHSIZE  (16 *  8)
#else
#define PREFETCHSIZE  (32 *  8)
#endif

#ifndef LN
#define CPREFETCHSIZE  7
#else
#define CPREFETCHSIZE -8
#endif
#define CPREFETCH     lfetch.excl.nt1

#define M	r32
#define N	r33
#define K	r34
#define A	r37
#define B	r38
#define C	r39
#define LDC	r35

#define I	r15
#define J	r16
#define AOFFSET	r17
#define BOFFSET	r18
#define TEMP	r19
#define L	r20

#define C1	r21
#define C2	r22
#define C3	r23
#define C4	r24
#define C5	r25
#define C6	r26
#define C7	r27
#define C8	r28

#define PREA	r8
#define PREB	r9
#define PREC	r10
#define SP	r12
#define ARLC	r29
#define PR	r30
#define ARPFS	r31

#define ALPHA_R	f8
#define ALPHA_I	f9

#define AORIG	loc0
#define KK	loc1
#define KK8	loc2
#define OFFSET	loc3
#define AOFFSET2 loc4
#define BOFFSET2 loc5

#ifndef CONJ
#define FCALC_A	  FSUB
#define FCALC_B	  FADD
#define FMA_A	  FNMA
#define FMA_B	  FMA
#else
#define FCALC_A	  FADD
#define FCALC_B	  FSUB
#define FMA_A	  FMA
#define FMA_B	  FNMA
#endif

#ifndef CONJ
#define FCALC_C	  FMA
#define FCALC_D	  FNMA
#else
#define FCALC_C	  FNMA
#define FCALC_D	  FMA
#endif

#ifndef CONJ
#define FMA_C	FNMA
#define FMA_D	FMA
#define FSUB_A	FSUB
#else
#define FMA_C	FMA
#define FMA_D	FMS
#define FSUB_A	FADD
#endif


	PROLOGUE
	.prologue
	PROFCODE

	{ .mfi
	.save	ar.pfs, ARPFS
	alloc	ARPFS = ar.pfs, 8, 8, 0, 0
	mov	f64  = f0
	adds	r14 = 16, SP
	}
	{ .mfi
	nop	__LINE__
	mov	f65  = f0
	adds	r15 = 24, SP
	}
	;;
	{ .mfi
	ld8	LDC    = [r14]
	mov	f81  = f0
	mov	PR = pr
	}
	{ .mfi
	ld8	OFFSET = [r15]
	mov	f96  = f0
	shr	J = N, 2
	}
	;;
	{ .mfi
	shladd	LDC = LDC, ZBASE_SHIFT, r0
	mov	f97  = f0
	}
	{ .mfi
	nop	__LINE__
	mov	f113 = f0
	}
	;;
#ifdef LN
       { .mmi
	setf.sig f32 = M
	setf.sig f33 = K
	shladd	C = M,  ZBASE_SHIFT, C
	}
	;;
	{.mmf
	nop	__LINE__
	nop	__LINE__
	xmpy.l	f32  = f32, f33
	}
	;;
	{ .mmi
	getf.sig r2 = f32
	;;
	nop	__LINE__
	shladd	A = r2, ZBASE_SHIFT, A
	}
	;;
#endif

#ifdef RN
	sub	KK  = r0, OFFSET
#endif

#ifdef RT
       { .mmi
	setf.sig f32 = N
	setf.sig f33 = K
	nop	 __LINE__
	}
	;;
	{ .mmi
	setf.sig f34 = LDC
	nop	 __LINE__
	nop	 __LINE__
	}
	;;
	{ .mmf
	nop	 __LINE__
	nop	 __LINE__
	xmpy.l	f33  = f32, f33
	}
	{ .mmf
	nop	 __LINE__
	sub	KK = N, OFFSET
	xmpy.l	f34  = f32, f34
	}
	;;
	{ .mmi
	getf.sig r2 = f33
	getf.sig r3 = f34
	}
	;;
	shladd	B = r2, ZBASE_SHIFT, B
	add	C = r3, C
#endif
	;;
	.body
	{ .mfi
	nop	__LINE__
	mov	f80  = f0
	mov	ARLC  = ar.lc
	}
	{ .mfb
	cmp.ge	p6, p0  = 0, J
	mov	f112 = f0
	(p6)	br.cond.dpnt .L050
	}
	;;
	.align 16

.L010:
#ifdef RT
       { .mmi
	shladd	r3 = LDC, 2, r0
	nop	__LINE__
	shl	r2 = K, 2 + ZBASE_SHIFT
	}
	;;
	{ .mmi
	sub	B = B, r2
	sub	C = C, r3
	nop	__LINE__
	}
	;;
#endif
	{ .mmi
	mov	C1 = C			// coffset1 = c + 0 * ldc
	add	C2 = LDC, C		// coffset2 = c + 1 * ldc
	}
	{ .mmi
	adds	J = -1, J
#ifdef LN
	add	KK = M, OFFSET
#elif defined LT
	mov	KK = OFFSET
#else
	nop	__LINE__
#endif
#if defined(LN) || defined(RT)
	mov	AORIG = A
#else
	mov	AOFFSET = A
#endif
	}
	;;
	{ .mmi
	shladd	C3 = LDC, 1, C		// coffset3 = c + 2 * ldc
	shladd	C4 = LDC, 1, C2		// coffset4 = c + 3 * ldc
#if defined(LT) || defined(RN)
	mov	L = KK
#else
	sub	L = K, KK
#endif
	}
	{ .mib
#ifndef RT
	shladd	C = LDC, 2, C		// coffset += 8 * ldc
#else
	nop	__LINE__
#endif
	tbit.z	p6, p7 = M, 0
	(p6)	br.cond.dptk .L020
	}
	;;
	{ .mmi
	cmp.ne	p7, p0 = r0, L
	adds	BOFFSET = 0 * SIZE, B
	shl	r2 = K, ZBASE_SHIFT
	}
	{ .mmi
	shladd	r3 = KK, ZBASE_SHIFT, r0
	nop	__LINE__
	nop	__LINE__
	}
	;;
#if defined(LT) || defined(RN)
	{ .mfb
	(p7) LDFPD	f48, f49 = [BOFFSET], 2 * SIZE
	mov	f72  = f0
	nop	__LINE__
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	mov	f73  = f0
	}
	;;
#else
	{ .mfi
	shladd	BOFFSET = r3, 2, B
	mov	f72  = f0
#ifdef LN
	sub	AORIG = AORIG, r2
#else
	nop	__LINE__
#endif
	}
	;;
	{ .mfi
	(p7) LDFPD	f48, f49 = [BOFFSET], 2 * SIZE
	mov	f73  = f0
	add	AOFFSET = r3, AORIG
	}
	;;
#endif
	;;
	adds	L =  1, L
	;;

	{ .mmi
	nop	__LINE__
	adds	PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
	tbit.z	p12, p0 = L, 0
	}
	;;
	{ .mfi
	(p7) LDFPD	f50, f51 = [BOFFSET], 2 * SIZE
	mov	f88  = f0
	shr	L = L, 1
	}
	{ .mfi
	(p7) LDFPD	f32, f33 = [AOFFSET], 2 * SIZE
	mov	f89  = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	(p7) LDFPD	f52, f53 = [BOFFSET], 2 * SIZE
	mov	f104  = f0
	adds	L =  -1, L
	}
	{ .mfb
	adds	PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
	mov	f105 = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	(p7) LDFPD	f54, f55 = [BOFFSET], 2 * SIZE	
	mov	f120 = f0
	mov	ar.lc = L
	}
	{ .mfi
	cmp.eq	p3, p0 = r0, r0
	mov	f121 = f0
	nop	__LINE__
	}
	;;
	cmp.eq  p6, p0 = -1, L
	(p6) br.cond.dpnt   .L038
	;;
	.align 16

.L032:
	{ .mfb
	lfetch.nt1	[PREA],  4 * SIZE
	FMA	f64   = f32, f48, f64	// A1 * B1
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	FMA_B	f65   = f32, f49, f65	// A1 * B2
	(p12) cmp.ne p3, p0 =  0, L
	}
	;;
	{ .mfi
	lfetch.nt1	[PREB],  16 * SIZE
	FMA	f80   = f32, f50, f80	// A1 * B3
	cmp.ne	p4, p5 =  0, L
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f81   = f32, f51, f81	// A1 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f56, f57 = [BOFFSET],   2 * SIZE
	FMA	f96   = f32, f52, f96	// A1 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f97  = f32, f53, f97	// A1 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f40, f41 = [AOFFSET], 2 * SIZE
	FMA	f112  = f32, f54, f112	// A1 * B7
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f113  = f32, f55, f113	// A1 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f58, f59 = [BOFFSET],  2 * SIZE
	FMA	f65   = f33, f48, f65	// A2 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f64   = f33, f49, f64	// A2 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f60, f61 = [BOFFSET], 2 * SIZE
	FMA	f81   = f33, f50, f81	// A2 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f80   = f33, f51, f80	// A2 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f62, f63 = [BOFFSET], 2 * SIZE
	FMA	f97   = f33, f52, f97	// A2 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f96  = f33, f53, f96	// A2 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f48, f49 = [BOFFSET],   2 * SIZE
	FMA	f113  = f33, f54, f113	// A2 * B7
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f112  = f33, f55, f112	// A2 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f32, f33 = [AOFFSET],   2 * SIZE
	(p3) FMA	f64   = f40, f56, f64	// A1 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f65   = f40, f57, f65	// A1 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f50, f51 = [BOFFSET],  2 * SIZE
	(p3) FMA	f80   = f40, f58, f80	// A1 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f81   = f40, f59, f81	// A1 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f52, f53 = [BOFFSET], 2 * SIZE
	(p3) FMA	f96   = f40, f60, f96	// A1 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f97  = f40, f61, f97	// A1 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	(p3) FMA	f112  = f40, f62, f112	// A1 * B7
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f113  = f40, f63, f113	// A1 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f54, f55 = [BOFFSET], 2 * SIZE
	(p3) FMA	f65   = f41, f56, f65	// A2 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_A	f64   = f41, f57, f64	// A2 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	(p3) FMA	f81   = f41, f58, f81	// A2 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_A	f80   = f41, f59, f80	// A2 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	(p3) FMA	f97   = f41, f60, f97	// A2 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_A	f96  = f41, f61, f96	// A2 * B6
	nop	__LINE__
	}
	;;
	{ .mfi
	nop	__LINE__
	(p3) FMA	f113  = f41, f62, f113	// A2 * B7
	adds	L = -1, L
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_A	f112  = f41, f63, f112	// A2 * B8
	br.cloop.sptk.few .L032
	}
	;;
.L038:
#if defined(LN) || defined(RT)
#ifdef LN
	adds	r2 = -1, KK
#else
	adds	r2 = -4, KK
#endif
	;;
	shladd	r2 = r2, ZBASE_SHIFT, r0
	;;
	add	AOFFSET = r2, AORIG
	shladd	BOFFSET = r2, 2, B
	;;	
#endif

#if defined(LN) || defined(LT)
	LDFPD	f72, f73 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f88, f89 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f104, f105 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f120, f121 = [BOFFSET]
	adds	BOFFSET = -6 * SIZE, BOFFSET
	;;
	FSUB	f64  = f72, f64
	FSUB_A	f65  = f73, f65
	FSUB	f80  = f88, f80
	FSUB_A	f81  = f89, f81
	FSUB	f96  = f104, f96
	FSUB_A	f97  = f105, f97
	FSUB	f112 = f120, f112
	FSUB_A	f113 = f121, f113
	;;
#else
	LDFPD	f72, f73 = [AOFFSET], 2 * SIZE
	;;
	LDFPD	f88, f89 = [AOFFSET], 2 * SIZE
	;;
	LDFPD	f104, f105 = [AOFFSET], 2 * SIZE
	;;
	LDFPD	f120, f121 = [AOFFSET]
	adds	AOFFSET = -6 * SIZE, AOFFSET
	;;
	FSUB	f64  = f72, f64
	FSUB	f65  = f73, f65
	FSUB	f80  = f88, f80
	FSUB	f81  = f89, f81
	FSUB	f96  = f104, f96
	FSUB	f97  = f105, f97
	FSUB	f112 = f120, f112
	FSUB	f113 = f121, f113
	;;
#endif

#ifdef LN
	LDFPD	f120, f121 = [AOFFSET]
	;;
	FMPY	f32  = f120, f64
	FMPY	f33  = f121, f64
	FMPY	f34  = f120, f80
	FMPY	f35  = f121, f80
	FMPY	f36  = f120, f96
	FMPY	f37  = f121, f96
	FMPY	f38  = f120, f112
	FMPY	f39  = f121, f112
	;;
	FMA_C	f64  = f121, f65,  f32
	FMA_D	f65  = f120, f65,  f33
	FMA_C	f80  = f121, f81,  f34
	FMA_D	f81  = f120, f81,  f35
	FMA_C	f96  = f121, f97,  f36
	FMA_D	f97  = f120, f97,  f37
	FMA_C	f112 = f121, f113, f38
	FMA_D	f113 = f120, f113, f39
	;;
#endif

#ifdef LT
	LDFPD	f90, f91 = [AOFFSET]
	;;
	FMPY	f32  = f90, f64
	FMPY	f33  = f91, f64
	FMPY	f34  = f90, f80
	FMPY	f35  = f91, f80
	FMPY	f36  = f90, f96
	FMPY	f37  = f91, f96
	FMPY	f38  = f90, f112
	FMPY	f39  = f91, f112
	;;
	FMA_C	f64  = f91, f65,  f32
	FMA_D	f65  = f90, f65,  f33
	FMA_C	f80  = f91, f81,  f34
	FMA_D	f81  = f90, f81,  f35
	FMA_C	f96  = f91, f97,  f36
	FMA_D	f97  = f90, f97,  f37
	FMA_C	f112 = f91, f113, f38
	FMA_D	f113 = f90, f113, f39
	;;
#endif

#ifdef RN
	LDFPD	f72, f73 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f74, f75 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f76, f77 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f78, f79 = [BOFFSET]
	adds	BOFFSET = 4 * SIZE, BOFFSET
	;;
	LDFPD	f90, f91 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f92, f93 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f94, f95 = [BOFFSET]
	adds	BOFFSET = 6 * SIZE, BOFFSET
	;;
	LDFPD	f108, f109 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f110, f111 = [BOFFSET]
	adds	BOFFSET = 8 * SIZE, BOFFSET
	;;
	LDFPD	f126, f127 = [BOFFSET]
	adds	BOFFSET = - 30 * SIZE, BOFFSET
	;;

	FMPY	f32  = f72, f64
	FMPY	f33  = f73, f64
	;;
	FMA_C	f64  = f73, f65, f32
	FMA_D	f65  = f72, f65, f33
	;;
	FNMA	f80  = f74, f64, f80
	FMA_A	f81  = f75, f64, f81
	;;
	FMA_B	f80  = f75, f65, f80
	FNMA	f81  = f74, f65, f81

	;;
	FNMA	f96  = f76, f64, f96
	FMA_A	f97  = f77, f64, f97
	;;
	FMA_B	f96  = f77, f65, f96
	FNMA	f97  = f76, f65, f97
	;;
	FNMA	f112 = f78, f64, f112
	FMA_A	f113 = f79, f64, f113
	;;
	FMA_B	f112 = f79, f65, f112
	FNMA	f113 = f78, f65, f113
	;;
	FMPY	f32  = f90, f80
	FMPY	f33  = f91, f80
	;;
	FMA_C	f80  = f91, f81,  f32
	FMA_D	f81  = f90, f81,  f33
	;;

	FNMA	f96  = f92, f80, f96
	FMA_A	f97  = f93, f80, f97
	;;
	FMA_B	f96  = f93, f81, f96
	FNMA	f97  = f92, f81, f97
	;;
	FNMA	f112 = f94, f80, f112
	FMA_A	f113 = f95, f80, f113
	;;
	FMA_B	f112 = f95, f81, f112
	FNMA	f113 = f94, f81, f113
	;;
	FMPY	f32  = f108, f96
	FMPY	f33  = f109, f96
	;;
	FMA_C	f96  = f109, f97,  f32
	FMA_D	f97  = f108, f97,  f33
	;;
	FNMA	f112 = f110, f96, f112
	FMA_A	f113 = f111, f96, f113
	;;
	FMA_B	f112 = f111, f97, f112
	FNMA	f113 = f110, f97, f113
	;;
	FMPY	f32  = f126, f112
	FMPY	f33  = f127, f112
	;;
	FMA_C	f112 = f127, f113, f32
	FMA_D	f113 = f126, f113, f33
	;;
#endif

#ifdef RT
	adds	BOFFSET = 30 * SIZE, BOFFSET
	;;
	LDFPD	f72, f73 = [BOFFSET]
	adds	BOFFSET = - 2 * SIZE, BOFFSET
	;;
	LDFPD	f74, f75 = [BOFFSET]
	adds	BOFFSET = - 2 * SIZE, BOFFSET
	;;
	LDFPD	f76, f77 = [BOFFSET]
	adds	BOFFSET = - 2 * SIZE, BOFFSET
	;;
	LDFPD	f78, f79 = [BOFFSET]
	adds	BOFFSET = - 4 * SIZE, BOFFSET
	;;
	LDFPD	f88, f89 = [BOFFSET]
	adds	BOFFSET = - 2 * SIZE, BOFFSET
	;;
	LDFPD	f90, f91 = [BOFFSET]
	adds	BOFFSET = - 2 * SIZE, BOFFSET
	;;
	LDFPD	f92, f93 = [BOFFSET]
	adds	BOFFSET = - 6 * SIZE, BOFFSET
	;;
	LDFPD	f104, f105 = [BOFFSET]
	adds	BOFFSET = - 2 * SIZE, BOFFSET
	;;
	LDFPD	f106, f107 = [BOFFSET]
	adds	BOFFSET = - 8 * SIZE, BOFFSET
	;;
	LDFPD	f120, f121 = [BOFFSET]
	;;
	FMPY	f32  = f72, f112
	FMPY	f33  = f73, f112
	;;
	FMA_C	f112 = f73, f113, f32
	FMA_D	f113 = f72, f113, f33
	;;
	FNMA	f96  = f74, f112, f96
	FMA_A	f97  = f75, f112, f97
	;;
	FMA_B	f96  = f75, f113, f96
	FNMA	f97  = f74, f113, f97
	;;
	FNMA	f80  = f76, f112, f80
	FMA_A	f81  = f77, f112, f81
	;;
	FMA_B	f80  = f77, f113, f80
	FNMA	f81  = f76, f113, f81
	;;
	FNMA	f64  = f78, f112, f64
	FMA_A	f65  = f79, f112, f65
	;;
	FMA_B	f64  = f79, f113, f64
	FNMA	f65  = f78, f113, f65
	;;
	FMPY	f32  = f88, f96
	FMPY	f33  = f89, f96
	;;
	FMA_C	f96  = f89, f97, f32
	FMA_D	f97  = f88, f97, f33
	;;
	FNMA	f80  = f90, f96, f80
	FMA_A	f81  = f91, f96, f81
	;;
	FMA_B	f80  = f91, f97, f80
	FNMA	f81  = f90, f97, f81
	;;
	FNMA	f64  = f92, f96, f64
	FMA_A	f65  = f93, f96, f65
	;;
	FMA_B	f64  = f93, f97, f64
	FNMA	f65  = f92, f97, f65
	;;
	FMPY	f32  = f104, f80
	FMPY	f33  = f105, f80
	;;
	FMA_C	f80  = f105, f81, f32
	FMA_D	f81  = f104, f81, f33
	;;
	FNMA	f64  = f106, f80, f64