gemm_kernel.s

Optimized GotoBLAS libraries

	FMA	f107 = ALPHA, f107, f59
	nop	__LINE__
	}
	{ .mfb
	STFD	[C13] = f103, 5 * SIZE
 	FMA	f111 = ALPHA, f111, f60
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C6 ] = f104, SIZE
	FMA	f112 = ALPHA, f112, f61
	nop	__LINE__
	}
	{ .mfb
	STFD	[C14] = f108, SIZE
	FMA	f116 = ALPHA, f116, f62
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C6 ] = f105, SIZE
	FMA	f113 = ALPHA, f113, f63
	nop	__LINE__
	}
	{ .mfb
	STFD	[C14] = f109, SIZE
	FMA	f117 = ALPHA, f117, f6
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C6 ] = f106, SIZE
	FMA	f114 = ALPHA, f114, f7
	nop	__LINE__
	}
	{ .mfb
	STFD	[C14] = f110, SIZE
	FMA	f118 = ALPHA, f118, f10
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C6 ] = f107, 5 * SIZE
	FMA	f115 = ALPHA, f115, f11
	nop	__LINE__
	}
	{ .mfb
	STFD	[C14] = f111, 5 * SIZE
	FMA	f119 = ALPHA, f119, f12
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C7 ] = f112, SIZE
	FMA	f120 = ALPHA, f120, f13
	nop	__LINE__
	}
	{ .mfb
	STFD	[C15] = f116, SIZE
	FMA	f124 = ALPHA, f124, f14
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C7 ] = f113, SIZE
	FMA	f121 = ALPHA, f121, f15
	nop	__LINE__
	}
	{ .mfb
	STFD	[C15] = f117, SIZE
	FMA	f125 = ALPHA, f125, f16
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C7 ] = f114, SIZE
	FMA	f122 = ALPHA, f122, f17
	nop	__LINE__
	}
	{ .mfb
	STFD	[C15] = f118, SIZE
	FMA	f126 = ALPHA, f126, f18
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C7 ] = f115, 5 * SIZE
	FMA	f123 = ALPHA, f123, f19
	nop	__LINE__
	}
	{ .mfb
	STFD	[C15] = f119, 5 * SIZE
	FMA	f127 = ALPHA, f127, f20
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C8 ] = f120, SIZE
	mov	f64  = f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C16] = f124, SIZE
	mov	f72  = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C8 ] = f121, SIZE
	mov	f80  = f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C16] = f125, SIZE
	mov	f88  = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C8 ] = f122, SIZE
	mov	f96  = f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C16] = f126, SIZE
	mov	f104 = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C8 ] = f123, 5 * SIZE
	mov	f112 = f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C16] = f127, 5 * SIZE
	mov	f120 = f0
	(p6)	br.cond.dptk .L011
	}
	;;
#else
	{ .mfi
	nop	__LINE__
	FMPY	f64  = ALPHA, f64
	cmp.ne	p6, p0 = 1, I
	}
	{ .mfb
	nop	__LINE__
	FMPY	f68  = ALPHA, f68
	nop	__LINE__
	}
	;;
	{ .mfi
	nop	__LINE__
	FMPY	f65  = ALPHA, f65
	adds	I = -1, I
	}
	{ .mfb
	nop	__LINE__
	FMPY	f69  = ALPHA, f69
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMPY	f66  = ALPHA, f66
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMPY	f70  = ALPHA, f70
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMPY	f67  = ALPHA, f67
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMPY	f71  = ALPHA, f71
	nop	__LINE__
	}
	;;
	{ .mmf
	STFD	[C1 ] = f64, SIZE
	STFD	[C9 ] = f68, SIZE
	FMPY	f72  = ALPHA, f72
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMPY	f76  = ALPHA, f76
	}
	;;
	{ .mmf
	STFD	[C1 ] = f65, SIZE
	STFD	[C9 ] = f69, SIZE
	FMPY	f73  = ALPHA, f73
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMPY	f77  = ALPHA, f77
	}
	;;
	{ .mmf
	STFD	[C1 ] = f66, SIZE
	STFD	[C9 ] = f70, SIZE
	FMPY	f74  = ALPHA, f74
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMPY	f78  = ALPHA, f78
	}
	;;
	{ .mfb
	STFD	[C1 ] = f67, 5 * SIZE
	FMPY	f75  = ALPHA, f75
	nop	__LINE__
	}
	{ .mfb
	STFD	[C9 ] = f71, 5 * SIZE
	FMPY	f79  = ALPHA, f79
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C2 ] = f72, SIZE
	FMPY	f80  = ALPHA, f80
	nop	__LINE__
	}
	{ .mfb
	STFD	[C10] = f76, SIZE
	FMPY	f84  = ALPHA, f84
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C2 ] = f73, SIZE
	FMPY	f81  = ALPHA, f81
	nop	__LINE__
	}
	{ .mfb
	STFD	[C10] = f77, SIZE
	FMPY	f85  = ALPHA, f85
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C2 ] = f74, SIZE
	FMPY	f82  = ALPHA, f82
	nop	__LINE__
	}
	{ .mfb
	STFD	[C10] = f78, SIZE
	FMPY	f86  = ALPHA, f86
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C2 ] = f75, 5 * SIZE
	FMPY	f83  = ALPHA, f83
	nop	__LINE__
	}
	{ .mfb
	STFD	[C10] = f79, 5 * SIZE
	FMPY	f87  = ALPHA, f87
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3 ] = f80, SIZE
	FMPY	f88  = ALPHA, f88
	nop	__LINE__
	}
	{ .mfb
	STFD	[C11] = f84, SIZE
	FMPY	f92  = ALPHA, f92
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3 ] = f81, SIZE
	FMPY	f89  = ALPHA, f89
	nop	__LINE__
	}
	{ .mfb
	STFD	[C11] = f85, SIZE
	FMPY	f93  = ALPHA, f93
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3 ] = f82, SIZE
	FMPY	f90  = ALPHA, f90
	nop	__LINE__
	}
	{ .mfb
	STFD	[C11] = f86, SIZE
	FMPY	f94  = ALPHA, f94
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3 ] = f83, 5 * SIZE
	FMPY	f91  = ALPHA, f91
	nop	__LINE__
	}
	{ .mfb
	STFD	[C11] = f87, 5 * SIZE
	FMPY	f95  = ALPHA, f95
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C4 ] = f88, SIZE
	FMPY	f96  = ALPHA, f96
	nop	__LINE__
	}
	{ .mfb
	STFD	[C12] = f92, SIZE
	FMPY	f100 = ALPHA, f100
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C4 ] = f89, SIZE
	FMPY	f97  = ALPHA, f97
	nop	__LINE__
	}
	{ .mfb
	STFD	[C12] = f93, SIZE
	FMPY	f101 = ALPHA, f101
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C4 ] = f90, SIZE
	FMPY	f98  = ALPHA, f98
	nop	__LINE__
	}
	{ .mfb
	STFD	[C12] = f94, SIZE
	FMPY	f102 = ALPHA, f102
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C4 ] = f91, 5 * SIZE
	FMPY	f99  = ALPHA, f99
	nop	__LINE__
	}
	{ .mfb
	STFD	[C12] = f95, 5 * SIZE
	FMPY	f103 = ALPHA, f103
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C5 ] = f96,  SIZE
	FMPY	f104 = ALPHA, f104
	nop	__LINE__
	}
	{ .mfb
	STFD	[C13] = f100, SIZE
	FMPY	f108 = ALPHA, f108
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C5 ] = f97,  SIZE
	FMPY	f105 = ALPHA, f105
	nop	__LINE__
	}
	{ .mfb
	STFD	[C13] = f101, SIZE
	FMPY	f109 = ALPHA, f109
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C5 ] = f98,  SIZE
	FMPY	f106 = ALPHA, f106
	nop	__LINE__
	}
	{ .mfb
	STFD	[C13] = f102, SIZE
	FMPY	f110 = ALPHA, f110
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C5 ] = f99,  5 * SIZE
	FMPY	f107 = ALPHA, f107
	nop	__LINE__
	}
	{ .mfb
	STFD	[C13] = f103, 5 * SIZE
 	FMPY	f111 = ALPHA, f111
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C6 ] = f104, SIZE
	FMPY	f112 = ALPHA, f112
	nop	__LINE__
	}
	{ .mfb
	STFD	[C14] = f108, SIZE
	FMPY	f116 = ALPHA, f116
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C6 ] = f105, SIZE
	FMPY	f113 = ALPHA, f113
	nop	__LINE__
	}
	{ .mfb
	STFD	[C14] = f109, SIZE
	FMPY	f117 = ALPHA, f117
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C6 ] = f106, SIZE
	FMPY	f114 = ALPHA, f114
	nop	__LINE__
	}
	{ .mfb
	STFD	[C14] = f110, SIZE
	FMPY	f118 = ALPHA, f118
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C6 ] = f107, 5 * SIZE
	FMPY	f115 = ALPHA, f115
	nop	__LINE__
	}
	{ .mfb
	STFD	[C14] = f111, 5 * SIZE
	FMPY	f119 = ALPHA, f119
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C7 ] = f112, SIZE
	FMPY	f120 = ALPHA, f120
	nop	__LINE__
	}
	{ .mfb
	STFD	[C15] = f116, SIZE
	FMPY	f124 = ALPHA, f124
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C7 ] = f113, SIZE
	FMPY	f121 = ALPHA, f121
	nop	__LINE__
	}
	{ .mfb
	STFD	[C15] = f117, SIZE
	FMPY	f125 = ALPHA, f125
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C7 ] = f114, SIZE
	FMPY	f122 = ALPHA, f122
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	sub	L = K, KK
#else
	nop	__LINE__
#endif
	}
	{ .mfb
	STFD	[C15] = f118, SIZE
	FMPY	f126 = ALPHA, f126
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C7 ] = f115, 5 * SIZE
	FMPY	f123 = ALPHA, f123
#if defined(TRMMKERNEL) && (defined(LEFT) &&  defined(TRANSA))
	adds	L = -8, L
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C15] = f119, 5 * SIZE
	FMPY	f127 = ALPHA, f127
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
	adds	L = -8, L
#else
	nop	__LINE__
#endif
	}
	;;
	{ .mfi
	STFD	[C8 ] = f120, SIZE
	mov	f64  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	KK8 = L, BASE_SHIFT, r0
#else
	nop	__LINE__
#endif
	}
	{ .mfb
	STFD	[C16] = f124, SIZE
	mov	f72  = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C8 ] = f121, SIZE
	mov	f80  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	AOFFSET = KK8, 3, AOFFSET
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C16] = f125, SIZE
	mov	f88  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	BOFFSET = KK8, 3, BOFFSET
#else
	nop	__LINE__
#endif
	}
	;;
	{ .mfi
	STFD	[C8 ] = f122, SIZE
	mov	f96  = f0
#if defined(TRMMKERNEL) && defined(LEFT)
	adds	KK = 8, KK
#else
	nop	__LINE__
#endif
	}
	{ .mfb
	STFD	[C16] = f126, SIZE
	mov	f104 = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C8 ] = f123, 5 * SIZE
	mov	f112 = f0
#ifdef TRMMKERNEL
	shladd	KK8 = KK, BASE_SHIFT, r0
#else
	nop	__LINE__
#endif
	}
	{ .mfb
	STFD	[C16] = f127, 5 * SIZE
	mov	f120 = f0
	(p6)	br.cond.dptk .L011
	}
	;;
#endif

.L020:
	{ .mfi
	cmp.eq	p3, p0 = r0, r0
	mov	f89  = f0
	tbit.z	p6, p7 = M, 2
	}
	{ .mfb
#ifndef TRMMKERNEL
	nop	__LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
	sub	L = K, KK
#elif defined(LEFT)
	adds	L = 4, KK
#else
	adds	L = 8, KK
#endif
#endif
	mov	f81  = f0
	(p6)	br.cond.dptk .L030
	}
	;;
#if !defined(TRMMKERNEL) || \
    defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	{ .mfi
	LDFPD	f48, f49 = [B]
	mov	f65  = f0
	nop	__LINE__
	}
	{ .mfi
	adds	BOFFSET = 2 * SIZE, B
	mov	f73  = f0
	adds	PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
	}
	;;
#else
	{ .mfi
	shladd	BOFFSET = KK8, 3, B
	mov	f65  = f0
	shladd	AOFFSET = KK8, 2, AOFFSET
	}
	;;
	{ .mfi
	LDFPD	f48, f49 = [BOFFSET], 2 * SIZE
	mov	f73  = f0
	adds	PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
	}
	;;
#endif
	{ .mmf
	LDFPD	f32, f33 = [AOFFSET], 2 * SIZE
	setf.d	f97  = r0
	mov	f105 = f0
	}
	{ .mfi
	setf.d	f113 = r0
	mov	f121 = f0
#ifndef TRMMKERNEL
	adds	L =  1, K
#else
	adds	L =  1, L
#endif
	}
	;;
	{ .mmf
	LDFPD	f50, f51 = [BOFFSET], 2 * SIZE
	setf.d	f66  = r0
	mov	f74  = f0
	}
	{ .mfi
	setf.d	f82  = r0
	mov	f90  = f0
	tbit.z	p12, p0 = L, 0
	}
	;;
	{ .mmf
	LDFPD	f52, f53 = [BOFFSET], 2 * SIZE
	setf.d	f98   = r0
	mov	f106  = f0
	}
	{ .mfi
	setf.d	f114 = r0
	mov	f122 = f0
	shr	L = L, 1
	}
	;;
	{ .mfi
	LDFPD	f54, f55 = [BOFFSET], 2 * SIZE
	mov	f75  = f0
	adds	L =  -1, L
	}
	{ .mmf
	setf.d	f67  = r0
	setf.d	f83  = r0
	mov	f91  = f0
	}
	;;
	{ .mfi
	LDFPD	f34, f35 = [AOFFSET], 2 * SIZE
	mov	f107 = f0
	mov	ar.lc = L
	}
	{ .mmf
	setf.d	f99  = r0
	setf.d	f115 = r0
	mov	f123 = f0
	}
	;;
	.align 32

.L022:
	{ .mfi
	lfetch.nt1	[PREA],  16 * SIZE
	FMA	f64   = f32, f48, f64	// A1 * B1
	adds	PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
	}
	{ .mfi
	nop	__LINE__
	FMA	f72   = f32, f49, f72	// A1 * B2
	(p12) cmp.ne p3, p0 =  0, L
	}
	;;
	{ .mfi
	lfetch.nt1	[PREB],  16 * SIZE
	FMA	f80   = f32, f50, f80	// A1 * B3
	cmp.ne	p4, p5 =  0, L
	}
	{ .mfb
	nop	__LINE__
	FMA	f88   = f32, f51, f88	// A1 * B4
	nop	__LINE__
	}
	;;
	{ .mfi
	(p3) LDFPD	f40, f41 = [AOFFSET], 2 * SIZE
	FMA	f96   = f32, f52, f96	// A1 * B5
	(p5) adds	C9  = 2 * SIZE, C1
	}
	{ .mfi
	nop	__LINE__
	FMA	f104  = f32, f53, f104	// A1 * B6
	(p5) adds	C10 = 2 * SIZE, C2
	}
	;;
	{ .mfi
	(p3) LDFPD	f56, f57 = [BOFFSET],   2 * SIZE
	FMA	f112  = f32, f54, f112	// A1 * B7
	(p5) adds	C11 = 2 * SIZE, C3
	}
	{ .mfi
	nop	__LINE__
	FMA	f120  = f32, f55, f120	// A1 * B8
	(p5) adds	C12 = 2 * SIZE, C4
	}
	;;
	{ .mfi
	(p3) LDFPD	f58, f59 = [BOFFSET],  2 * SIZE
	FMA	f65   = f33, f48, f65	// A2 * B1
	(p5) adds	C13 = 2 * SIZE, C5
	}
	{ .mfi
	nop	__LINE__
	FMA	f73   = f33, f49, f73	// A2 * B2
	(p5) adds	C14 = 2 * SIZE, C6
	}
	;;
	{ .mfi
	(p3) LDFPD	f60, f61 = [BOFFSET], 2 * SIZE
	FMA	f81   = f33, f50, f81	// A2 * B3
	(p5) adds	C15 = 2 * SIZE, C7
	}
	{ .mfi
	nop	__LINE__
	FMA	f89   = f33, f51, f89	// A2 * B4
	(p5) adds	C16 = 2 * SIZE, C8
	}
	;;
	{ .mfb
	(p3) LDFPD	f62, f63 = [BOFFSET], 2 * SIZE
	FMA	f97   = f33, f52, f97	// A2 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f105  = f33, f53, f105	// A2 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f42, f43 = [AOFFSET], 2 * SIZE
	FMA	f113  = f33, f54, f113	// A2 * B7
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f121  = f33, f55, f121	// A2 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f66   = f34, f48, f66	// A3 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f74   = f34, f49, f74	// A3 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f82   = f34, f50, f82	// A3 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f90   = f34, f51, f90	// A3 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f98   = f34, f52, f98	// A3 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f106  = f34, f53, f106	// A3 * B6
	nop	__LINE__
	}