zgemm_kernel.s

Optimized GotoBLAS libraries

	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f72  = ALPHA_I, f65, f72
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f76 = ALPHA_I, f69, f76
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f73  = ALPHA_I, f64, f73
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f77 = ALPHA_I, f68, f77
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f74 = ALPHA_I, f67, f74
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f78 = ALPHA_I, f71, f78
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f75 = ALPHA_I, f66, f75
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f79 = ALPHA_I, f70, f79
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f72,  SIZE
	FMPY	f88 = ALPHA_R, f80
	nop	__LINE__
	}
	{ .mfb
	STFD	[C5] = f76, SIZE
	FMPY	f92 = ALPHA_R, f84
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f73,  SIZE
	FCALC_C	f89 = ALPHA_R, f81, f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C5] = f77, SIZE
	FCALC_C	f93 = ALPHA_R, f85, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f74, SIZE
	FMPY	f90 = ALPHA_R, f82
	nop	__LINE__
	}
	{ .mfb
	STFD	[C5] = f78, SIZE
	FMPY	f94 = ALPHA_R, f86
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f75, 5 * SIZE
	FCALC_C	f91 = ALPHA_R, f83, f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C5] = f79, 5 * SIZE
	FCALC_C	f95 = ALPHA_R, f87, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f88 = ALPHA_I, f81, f88
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f92 = ALPHA_I, f85, f92
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f89 = ALPHA_I, f80, f89
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f93 = ALPHA_I, f84, f93
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f90 = ALPHA_I, f83, f90
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f94 = ALPHA_I, f87, f94
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f91 = ALPHA_I, f82, f91
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f95 = ALPHA_I, f86, f95
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C2] = f88, SIZE
	FMPY	f104 = ALPHA_R, f96
	nop	__LINE__
	}
	{ .mfb
	STFD	[C6] = f92, SIZE
	FMPY	f108 = ALPHA_R, f100
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C2] = f89, SIZE
	FCALC_C	f105 = ALPHA_R, f97,  f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C6] = f93, SIZE
	FCALC_C	f109 = ALPHA_R, f101, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C2] = f90, SIZE
	FMPY	f106 = ALPHA_R, f98
	nop	__LINE__
	}
	{ .mfb
	STFD	[C6] = f94, SIZE
	FMPY	f110 = ALPHA_R, f102
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C2] = f91, 5 * SIZE
	FCALC_C	f107 = ALPHA_R, f99,  f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C6] = f95, 5 * SIZE
	FCALC_C	f111 = ALPHA_R, f103, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f104 = ALPHA_I, f97,  f104
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f108 = ALPHA_I, f101, f108
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f105 = ALPHA_I, f96,  f105
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f109 = ALPHA_I, f100, f109
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f106 = ALPHA_I, f99,  f106
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
 	FCALC_D	f110 = ALPHA_I, f103, f110
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f107 = ALPHA_I, f98,  f107
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f111 = ALPHA_I, f102, f111
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3] = f104, SIZE
	FMPY	f120 = ALPHA_R, f112
	nop	__LINE__
	}
	{ .mfb
	STFD	[C7] = f108, SIZE
	FMPY	f124 = ALPHA_R, f116
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3] = f105, SIZE
	FCALC_C	f121 = ALPHA_R, f113, f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C7] = f109, SIZE
	FCALC_C	f125 = ALPHA_R, f117, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3] = f106, SIZE
	FMPY	f122 = ALPHA_R, f114
	nop	__LINE__
	}
	{ .mfb
	STFD	[C7] = f110, SIZE
	FMPY	f126 = ALPHA_R, f118
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3] = f107, 5 * SIZE
	FCALC_C	f123 = ALPHA_R, f115, f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C7] = f111, 5 * SIZE
	FCALC_C	f127 = ALPHA_R, f119, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f120 = ALPHA_I, f113, f120
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f124 = ALPHA_I, f117, f124
	nop	__LINE__
	}
	;;
	{ .mfi
	nop	__LINE__
	FMA	f121 = ALPHA_I, f112, f121
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	sub	L = K, KK
#else
	nop	__LINE__
#endif
	}
	{ .mfb
	nop	__LINE__
	FMA	f125 = ALPHA_I, f116, f125
	nop	__LINE__
	}
	;;
	{ .mfi
	nop	__LINE__
	FCALC_D	f122 = ALPHA_I, f115, f122
#if defined(TRMMKERNEL) && (defined(LEFT) &&  defined(TRANSA))
	adds	L = -4, L
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	nop	__LINE__
	FCALC_D	f126 = ALPHA_I, f119, f126
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
	adds	L = -4, L
#else
	nop	__LINE__
#endif
	}
	;;
	{ .mfi
	nop	__LINE__
	FMA	f123 = ALPHA_I, f114, f123
	cmp.ne	p6, p0 = 1, I
	}
	{ .mfi
	nop	__LINE__
	FMA	f127 = ALPHA_I, f118, f127
	adds	I = -1, I
	}
	;;
	{ .mfi
	STFD	[C4] = f120, SIZE
	mov	f64  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	KK8 = L, ZBASE_SHIFT, r0
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C8] = f124, SIZE
	mov	f65  = f0
	and	TEMP = 3, M
	}
	;;
	{ .mfi
	STFD	[C4] = f121, SIZE
	mov	f80  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	AOFFSET = KK8, 2, AOFFSET
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C8] = f125, SIZE
	mov	f81  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	BOFFSET = KK8, 2, BOFFSET
#else
	nop	__LINE__
#endif
	}
	;;
	{ .mfi
	STFD	[C4] = f122, SIZE
	mov	f96  = f0
#if defined(TRMMKERNEL) && defined(LEFT)
	adds	KK = 4, KK
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C8] = f126, SIZE
	mov	f97  = f0
	cmp.ne	p8, p9 = r0, TEMP
	}
	;;
	{ .mfi
	STFD	[C4] = f123, 5 * SIZE
	mov	f112 = f0
#ifdef TRMMKERNEL
	shladd	KK8 = KK, ZBASE_SHIFT, r0
#else
	nop	__LINE__
#endif
	}
	{ .mfb
	STFD	[C8] = f127, 5 * SIZE
	mov	f113  = f0
	(p6)	br.cond.dptk .L011
	}
	;;
#endif

.L020:
	{ .mib
#ifndef TRMMKERNEL
	nop	__LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
	sub	L = K, KK
#elif defined(LEFT)
	adds	L = 2, KK
#else
	adds	L = 4, KK
#endif
#endif
	tbit.z	p6, p7 = M, 1
	(p6)	br.cond.dptk .L030
	}
	;;
#if !defined(TRMMKERNEL) || \
    defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	{ .mfb
	LDFPD	f48, f49 = [B]
	mov	f66 = f0
	nop	__LINE__
	}
	{ .mfi
	adds	BOFFSET = 2 * SIZE, B
	mov	f67 = f0
#ifndef TRMMKERNEL
	adds	L =  1, K
#else
	adds	L =  1, L
#endif
	}
	;;
#else
	{ .mfi
	shladd	BOFFSET = KK8, 2, B
	mov	f66 = f0
	shladd	AOFFSET = KK8, 1, AOFFSET
	}
	;;
	{ .mfi
	LDFPD	f48, f49 = [BOFFSET], 2 * SIZE
	mov	f67 = f0
#ifndef TRMMKERNEL
	adds	L =  1, K
#else
	adds	L =  1, L
#endif
	}
	;;
#endif
	{ .mfi
	LDFPD	f32, f33 = [AOFFSET], 2 * SIZE
	mov	f82 = f0
	tbit.z	p12, p0 = L, 0
	}
	{ .mfi	
	LDFPD	f50, f51 = [BOFFSET], 2 * SIZE
	mov	f83 = f0
	shr	L = L, 1
	}
	;;
	{ .mfi
	LDFPD	f34, f35 = [AOFFSET], 2 * SIZE
	mov	f98 = f0
	adds	L =  -1, L
	}
	{ .mfi
	LDFPD	f52, f53 = [BOFFSET], 2 * SIZE
	mov	f99 = f0
	cmp.eq	p3, p0 = r0, r0
	}
	;;
	{ .mfi
	LDFPD	f54, f55 = [BOFFSET], 2 * SIZE
	mov	f114 = f0
	mov	ar.lc = L
	}
	{ .mfi	
	adds	PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
	mov	f115 = f0
	nop	__LINE__
	}
	;;
	.align 16

.L022:
	{ .mfi
	lfetch.nt1	[PREA],  8 * SIZE
	FMA	f64   = f32, f48, f64	// A1 * B1
	adds	PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
	}
	{ .mfi
	nop	__LINE__
	FMA_B	f65   = f32, f49, f65	// A1 * B2
	(p12) cmp.ne p3, p0 =  0, L
	}
	;;
	{ .mfi
	lfetch.nt1	[PREB],  16 * SIZE
	FMA	f80   = f32, f50, f80	// A1 * B3
	cmp.ne	p4, p5 =  0, L
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f81   = f32, f51, f81	// A1 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f40, f41 = [AOFFSET], 2 * SIZE
	FMA	f96   = f32, f52, f96	// A1 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f97  = f32, f53, f97	// A1 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f56, f57 = [BOFFSET],   2 * SIZE
	FMA	f112  = f32, f54, f112	// A1 * B7
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f113  = f32, f55, f113	// A1 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f58, f59 = [BOFFSET],  2 * SIZE
	FMA	f65   = f33, f48, f65	// A2 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f64   = f33, f49, f64	// A2 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f60, f61 = [BOFFSET], 2 * SIZE
	FMA	f81   = f33, f50, f81	// A2 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f80   = f33, f51, f80	// A2 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f62, f63 = [BOFFSET], 2 * SIZE
	FMA	f97   = f33, f52, f97	// A2 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f96  = f33, f53, f96	// A2 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f42, f43 = [AOFFSET], 2 * SIZE
	FMA	f113  = f33, f54, f113	// A2 * B7
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f112  = f33, f55, f112	// A2 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f66   = f34, f48, f66	// A3 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f67   = f34, f49, f67	// A3 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f82   = f34, f50, f82	// A3 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f83   = f34, f51, f83	// A3 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f98   = f34, f52, f98	// A3 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f99  = f34, f53, f99	// A3 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f114  = f34, f54, f114	// A3 * B7
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f115  = f34, f55, f115	// A3 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f67   = f35, f48, f67	// A4 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f66   = f35, f49, f66	// A4 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f83   = f35, f50, f83	// A4 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f82   = f35, f51, f82	// A4 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f32, f33 = [AOFFSET],   2 * SIZE
	FMA	f99   = f35, f52, f99	// A4 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f98  = f35, f53, f98	// A4 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f48, f49 = [BOFFSET],   2 * SIZE
	FMA	f115  = f35, f54, f115	// A4 * B7
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f114  = f35, f55, f114	// A4 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f50, f51 = [BOFFSET],  2 * SIZE
	(p3) FMA	f64   = f40, f56, f64	// A1 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f65   = f40, f57, f65	// A1 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f52, f53 = [BOFFSET], 2 * SIZE
	(p3) FMA	f80   = f40, f58, f80	// A1 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f81   = f40, f59, f81	// A1 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f72  = [C1], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f96   = f40, f60, f96	// A1 * B5
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f88 = [C2], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_B	f97  = f40, f61, f97	// A1 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f73  = [C1], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f112  = f40, f62, f112	// A1 * B7
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f89 = [C2], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_B	f113  = f40, f63, f113	// A1 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f54, f55 = [BOFFSET], 2 * SIZE
	(p3) FMA	f65   = f41, f56, f65	// A2 * B1
	nop	__LINE__
	}
	{ .mfb
	(p3) FMA_A	f64   = f41, f57, f64	// A2 * B2
	nop	__LINE__
	}
	{ .mfb
	(p4) LDFPD	f34, f35 = [AOFFSET], 2 * SIZE
	(p3) FMA	f81   = f41, f58, f81	// A2 * B3
	nop	__LINE__
	}
	{ .mfb
	(p3) FMA_A	f80   = f41, f59, f80	// A2 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f74 = [C1], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f97   = f41, f60, f97	// A2 * B5
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f90 = [C2], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_A	f96  = f41, f61, f96	// A2 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f75 = [C1], -3 * SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f113  = f41, f62, f113	// A2 * B7
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f91 = [C2], -3 * SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_A	f112  = f41, f63, f112	// A2 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f104 = [C3], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f66   = f42, f56, f66	// A3 * B1
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f120 = [C4], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_B	f67   = f42, f57, f67	// A3 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f105 = [C3], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f82   = f42, f58, f82	// A3 * B3
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f121 = [C4], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_B	f83   = f42, f59, f83	// A3 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f106 = [C3], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f98   = f42, f60, f98	// A3 * B5
	nop	__LINE__
	}
	{ .mfb