zgemm_kernel.s

Optimized GotoBLAS libraries

#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f122 = [C4], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_B	f99  = f42, f61, f99	// A3 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f107 = [C3], -3 * SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f114  = f42, f62, f114	// A3 * B7
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f123 = [C4], -3 * SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_B	f115  = f42, f63, f115	// A3 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	(p3) FMA	f67   = f43, f56, f67	// A4 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_A	f66   = f43, f57, f66	// A4 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	(p3) FMA	f83   = f43, f58, f83	// A4 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_A	f82   = f43, f59, f82	// A4 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	(p3) FMA	f99   = f43, f60, f99	// A4 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_A	f98  = f43, f61, f98	// A4 * B6
	nop	__LINE__
	}
	;;
	{ .mfi
	nop	__LINE__
	(p3) FMA	f115  = f43, f62, f115	// A4 * B7
	adds	L = -1, L
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_A	f114  = f43, f63, f114	// A4 * B8
	br.cloop.sptk.few .L022
	}
	;;
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	{ .mfb
	nop	__LINE__
	FMA	f72  = ALPHA_R, f64, f72
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f88 = ALPHA_R, f80, f88
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_C	f73  = ALPHA_R, f65, f73
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_C	f89 = ALPHA_R, f81, f89
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f74 = ALPHA_R, f66, f74
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f90 = ALPHA_R, f82, f90
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_C	f75 = ALPHA_R, f67, f75
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_C	f91 = ALPHA_R, f83, f91
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f72  = ALPHA_I, f65, f72
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f88 = ALPHA_I, f81,  f88
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f73  = ALPHA_I, f64, f73
	nop	__LINE__
	}
	{ .mfb
	FMA	f89 = ALPHA_I, f80,  f89
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f74 = ALPHA_I, f67, f74
	nop	__LINE__
	}
	{ .mfb
	FCALC_D	f90 = ALPHA_I, f83,  f90
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f75 = ALPHA_I, f66, f75
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f91 = ALPHA_I, f82,  f91
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f72, SIZE
	FMA	f104 = ALPHA_R, f96,  f104
	nop	__LINE__
	}
	{ .mfb
	STFD	[C2] = f88, SIZE
	FMA	f120 = ALPHA_R, f112, f120
	nop	__LINE__
	}
	;;	
	{ .mfb
	STFD	[C1] = f73, SIZE
	FCALC_C	f105 = ALPHA_R, f97,  f105
	nop	__LINE__
	}
	{ .mfb
	STFD	[C2] = f89, SIZE
	FCALC_C	f121 = ALPHA_R, f113, f121
	nop	__LINE__
	}
	;;	
	{ .mfb
	STFD	[C1] = f74, SIZE
	FMA	f106 = ALPHA_R, f98,  f106
	nop	__LINE__
	}
	{ .mfb
	STFD	[C2] = f90, SIZE
	FMA	f122 = ALPHA_R, f114, f122
	nop	__LINE__
	}
	;;	
	{ .mfb
	STFD	[C1] = f75, SIZE
	FCALC_C	f107 = ALPHA_R, f99,  f107
	nop	__LINE__
	}
	{ .mfb
	STFD	[C2] = f91, SIZE
	FCALC_C	f123 = ALPHA_R, f115, f123
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f104 = ALPHA_I, f97,  f104
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f120 = ALPHA_I, f113, f120
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f105 = ALPHA_I, f96,  f105
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f121 = ALPHA_I, f112, f121
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f106 = ALPHA_I, f99,  f106
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f122 = ALPHA_I, f115, f122
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f107 = ALPHA_I, f98,  f107
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f123 = ALPHA_I, f114, f123
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3] = f104, SIZE
	mov	f64  = f0
	nop	__LINE__
	}
	{ .mfi
	STFD	[C4] = f120, SIZE
	mov	f65  = f0
	}
	;;
	{ .mfb
	STFD	[C3] = f105, SIZE
	mov	f80  = f0
	nop	__LINE__
	}
	{ .mfi
	STFD	[C4] = f121, SIZE
	mov	f81  = f0
	}
	;;
	{ .mfb
	STFD	[C3] = f106, SIZE
	mov	f96  = f0
	nop	__LINE__
	}
	{ .mfi
	STFD	[C4] = f122, SIZE
	mov	f97 = f0
	}
	;;
	{ .mfi
	STFD	[C3] = f107, SIZE
	mov	f112 = f0
	}
	{ .mfb
	STFD	[C4] = f123, SIZE
	mov	f113 = f0
	nop	__LINE__
	}
	;;
#else
	{ .mfb
	nop	__LINE__
	FMPY	f72  = ALPHA_R, f64
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMPY	f88 = ALPHA_R, f80
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_C	f73  = ALPHA_R, f65, f0
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_C	f89 = ALPHA_R, f81, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMPY	f74 = ALPHA_R, f66
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMPY	f90 = ALPHA_R, f82
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_C	f75 = ALPHA_R, f67, f0
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_C	f91 = ALPHA_R, f83, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f72  = ALPHA_I, f65, f72
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f88 = ALPHA_I, f81,  f88
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f73  = ALPHA_I, f64, f73
	nop	__LINE__
	}
	{ .mfb
	FMA	f89 = ALPHA_I, f80,  f89
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f74 = ALPHA_I, f67, f74
	nop	__LINE__
	}
	{ .mfb
	FCALC_D	f90 = ALPHA_I, f83,  f90
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f75 = ALPHA_I, f66, f75
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f91 = ALPHA_I, f82,  f91
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f72, SIZE
	FMPY	f104 = ALPHA_R, f96
	nop	__LINE__
	}
	{ .mfb
	STFD	[C2] = f88, SIZE
	FMPY	f120 = ALPHA_R, f112
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f73, SIZE
	FCALC_C	f105 = ALPHA_R, f97,  f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C2] = f89, SIZE
	FCALC_C	f121 = ALPHA_R, f113, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f74, SIZE
	FMPY	f106 = ALPHA_R, f98
	nop	__LINE__
	}
	{ .mfb
	STFD	[C2] = f90, SIZE
	FMPY	f122 = ALPHA_R, f114
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f75, SIZE
	FCALC_C	f107 = ALPHA_R, f99,  f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C2] = f91, SIZE
	FCALC_C	f123 = ALPHA_R, f115, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f104 = ALPHA_I, f97,  f104
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f120 = ALPHA_I, f113, f120
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f105 = ALPHA_I, f96,  f105
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f121 = ALPHA_I, f112, f121
	nop	__LINE__
	}
	;;
	{ .mfi
	nop	__LINE__
	FCALC_D	f106 = ALPHA_I, f99,  f106
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	sub	L = K, KK
#else
	nop	__LINE__
#endif
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f122 = ALPHA_I, f115, f122
	nop	__LINE__
	}
	;;
	{ .mfi
	nop	__LINE__
	FMA	f107 = ALPHA_I, f98,  f107
#if defined(TRMMKERNEL) && (defined(LEFT) &&  defined(TRANSA))
	adds	L = -2, L
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	nop	__LINE__
	FMA	f123 = ALPHA_I, f114, f123
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
	adds	L = -4, L
#else
	nop	__LINE__
#endif
	}
	;;
	{ .mfi
	STFD	[C3] = f104, SIZE
	mov	f64  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	KK8 = L, ZBASE_SHIFT, r0
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C4] = f120, SIZE
	mov	f65  = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C3] = f105, SIZE
	mov	f80  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	AOFFSET = KK8, 1, AOFFSET
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C4] = f121, SIZE
	mov	f81  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	BOFFSET = KK8, 2, BOFFSET
#else
	nop	__LINE__
#endif
	}
	;;
	{ .mfi
	STFD	[C3] = f106, SIZE
	mov	f96  = f0
#if defined(TRMMKERNEL) && defined(LEFT)
	adds	KK = 2, KK
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C4] = f122, SIZE
	mov	f97 = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C3] = f107, SIZE
	mov	f112 = f0
#ifdef TRMMKERNEL
	shladd	KK8 = KK, ZBASE_SHIFT, r0
#else
	nop	__LINE__
#endif
	}
	{ .mfb
	STFD	[C4] = f123, SIZE
	mov	f113 = f0
	nop	__LINE__
	}
	;;
#endif
	.align 16

.L030:
	{ .mib
#ifndef TRMMKERNEL
	nop	__LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
	sub	L = K, KK
#elif defined(LEFT)
	adds	L = 1, KK
#else
	adds	L = 4, KK
#endif
#endif
	tbit.z	p6, p7 = M, 0
	(p6)	br.cond.dptk .L049
	}
	;;
#if !defined(TRMMKERNEL) || \
    defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	{ .mfb
	LDFPD	f48, f49 = [B]
	mov	f72  = f0
	nop	__LINE__
	}
	{ .mfi
	adds	BOFFSET = 2 * SIZE, B
	mov	f73  = f0
#ifndef TRMMKERNEL
	adds	L =  1, K
#else
	adds	L =  1, L
#endif
	}
	;;
#else
	{ .mfi
	shladd	BOFFSET = KK8, 2, B
	mov	f72  = f0
	add	AOFFSET = KK8, AOFFSET
	}
	;;
	{ .mfi
	LDFPD	f48, f49 = [BOFFSET], 2 * SIZE
	mov	f73  = f0
#ifndef TRMMKERNEL
	adds	L =  1, K
#else
	adds	L =  1, L
#endif
	}
	;;
#endif
	{ .mmi
	nop	__LINE__
	adds	PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
	tbit.z	p12, p0 = L, 0
	}
	;;
	{ .mfi
	LDFPD	f50, f51 = [BOFFSET], 2 * SIZE
	mov	f88  = f0
	shr	L = L, 1
	}
	{ .mfi
	(p7) LDFPD	f32, f33 = [AOFFSET], 2 * SIZE
	mov	f89  = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	LDFPD	f52, f53 = [BOFFSET], 2 * SIZE
	mov	f104  = f0
	adds	L =  -1, L
	}
	{ .mfb
	adds	PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
	mov	f105 = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	LDFPD	f54, f55 = [BOFFSET], 2 * SIZE	
	mov	f120 = f0
	mov	ar.lc = L
	}
	{ .mfi
	cmp.eq	p3, p0 = r0, r0
	mov	f121 = f0
	nop	__LINE__
	}
	;;
	.align 16

.L032:
	{ .mfb
	lfetch.nt1	[PREA],  4 * SIZE
	FMA	f64   = f32, f48, f64	// A1 * B1
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	FMA_B	f65   = f32, f49, f65	// A1 * B2
	(p12) cmp.ne p3, p0 =  0, L
	}
	;;
	{ .mfi
	lfetch.nt1	[PREB],  16 * SIZE
	FMA	f80   = f32, f50, f80	// A1 * B3
	cmp.ne	p4, p5 =  0, L
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f81   = f32, f51, f81	// A1 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f56, f57 = [BOFFSET],   2 * SIZE
	FMA	f96   = f32, f52, f96	// A1 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f97  = f32, f53, f97	// A1 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f40, f41 = [AOFFSET], 2 * SIZE
	FMA	f112  = f32, f54, f112	// A1 * B7
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f113  = f32, f55, f113	// A1 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f58, f59 = [BOFFSET],  2 * SIZE
	FMA	f65   = f33, f48, f65	// A2 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f64   = f33, f49, f64	// A2 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f60, f61 = [BOFFSET], 2 * SIZE
	FMA	f81   = f33, f50, f81	// A2 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f80   = f33, f51, f80	// A2 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f62, f63 = [BOFFSET], 2 * SIZE
	FMA	f97   = f33, f52, f97	// A2 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f96  = f33, f53, f96	// A2 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f48, f49 = [BOFFSET],   2 * SIZE
	FMA	f113  = f33, f54, f113	// A2 * B7
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f112  = f33, f55, f112	// A2 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f32, f33 = [AOFFSET],   2 * SIZE
	(p3) FMA	f64   = f40, f56, f64	// A1 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f65   = f40, f57, f65	// A1 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f50, f51 = [BOFFSET],  2 * SIZE
	(p3) FMA	f80   = f40, f58, f80	// A1 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f81   = f40, f59, f81	// A1 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f52, f53 = [BOFFSET], 2 * SIZE
	(p3) FMA	f96   = f40, f60, f96	// A1 * B5
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f97  = f40, f61, f97	// A1 * B6
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f72  = [C1], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f112  = f40, f62, f112	// A1 * B7
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f88 = [C2], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_B	f113  = f40, f63, f113	// A1 * B8
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f54, f55 = [BOFFSET], 2 * SIZE
	(p3) FMA	f65   = f41, f56, f65	// A2 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_A	f64   = f41, f57, f64	// A2 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f73  = [C1], - SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f81   = f41, f58, f81	// A2 * B3
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f89 = [C2], - SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_A	f80   = f41, f59, f80	// A2 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f104 = [C3], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f97   = f41, f60, f97	// A2 * B5
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f120 = [C4], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_A	f96  = f41, f61, f96	// A2 * B6