zgemm_kernel.s

Optimized GotoBLAS libraries

	nop	__LINE__
	}
	;;
	{ .mfi
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f105 = [C3], - SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f113  = f41, f62, f113	// A2 * B7
	adds	L = -1, L
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f121 = [C4], - SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_A	f112  = f41, f63, f112	// A2 * B8
	br.cloop.sptk.few .L032
	}
	;;
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	{ .mfb
	nop	__LINE__
	FMA	f72  = ALPHA_R, f64, f72
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f88 = ALPHA_R, f80, f88
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_C	f73  = ALPHA_R, f65, f73
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_C	f89 = ALPHA_R, f81, f89
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f104 = ALPHA_R, f96,  f104
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f120 = ALPHA_R, f112, f120
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_C	f105 = ALPHA_R, f97,  f105
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_C	f121 = ALPHA_R, f113, f121
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f72  = ALPHA_I, f65, f72
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f88 = ALPHA_I, f81,  f88
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f73  = ALPHA_I, f64, f73
	nop	__LINE__
	}
	{ .mfb
	FMA	f89 = ALPHA_I, f80,  f89
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f104 = ALPHA_I, f97,  f104
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f120 = ALPHA_I, f113, f120
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f105 = ALPHA_I, f96,  f105
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f121 = ALPHA_I, f112, f121
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f72,  SIZE
	mov	f64  = f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C2] = f88, SIZE
	mov	f65  = f0
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C1] = f73,  SIZE
	mov	f80  = f0
	nop	__LINE__
	}
	{ .mfb
	STFD	[C2] = f89, SIZE
	mov	f81  = f0
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3] = f104, SIZE
	mov	f96  = f0
	nop	__LINE__
	}
	{ .mfi
	STFD	[C4] = f120, SIZE
	mov	f97 = f0
	nop	__LINE__
	}
	;;
	{ .mfb
	STFD	[C3] = f105, SIZE
	mov	f112 = f0
	nop	__LINE__
	}
	{ .mfi
	STFD	[C4] = f121, SIZE
	mov	f113 = f0
	nop	__LINE__
	}
	;;
#else
	{ .mfb
	nop	__LINE__
	FMA	f72  = ALPHA_R, f64, f0
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f88 = ALPHA_R, f80, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_C	f73  = ALPHA_R, f65, f0
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_C	f89 = ALPHA_R, f81, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f104 = ALPHA_R, f96,  f0
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA	f120 = ALPHA_R, f112, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_C	f105 = ALPHA_R, f97,  f0
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_C	f121 = ALPHA_R, f113, f0
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_D	f72  = ALPHA_I, f65, f72
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f88 = ALPHA_I, f81,  f88
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f73  = ALPHA_I, f64, f73
	nop	__LINE__
	}
	{ .mfb
	FMA	f89 = ALPHA_I, f80,  f89
	nop	__LINE__
	}
	;;
	{ .mfi
	nop	__LINE__
	FCALC_D	f104 = ALPHA_I, f97,  f104
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	sub	L = K, KK
#else
	nop	__LINE__
#endif
	}
	{ .mfb
	nop	__LINE__
	FCALC_D	f120 = ALPHA_I, f113, f120
	nop	__LINE__
	}
	;;
	{ .mfi
	nop	__LINE__
	FMA	f105 = ALPHA_I, f96,  f105
#if defined(TRMMKERNEL) && (defined(LEFT) &&  defined(TRANSA))
	adds	L = -1, L
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	nop	__LINE__
	FMA	f121 = ALPHA_I, f112, f121
#if defined(TRMMKERNEL) && (!defined(LEFT) && !defined(TRANSA))
	adds	L = -4, L
#else
	nop	__LINE__
#endif
	}
	;;
	{ .mfi
	STFD	[C1] = f72,  SIZE
	mov	f64  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	KK8 = L, ZBASE_SHIFT, r0
#else
	nop	__LINE__
#endif
	}
	{ .mfb
	STFD	[C2] = f88, SIZE
	mov	f65  = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C1] = f73,  SIZE
	mov	f80  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	add	AOFFSET = KK8, AOFFSET
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C2] = f89, SIZE
	mov	f81  = f0
#if defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	shladd	BOFFSET = KK8, 2, BOFFSET
#else
	nop	__LINE__
#endif
	}
	;;
	{ .mfi
	STFD	[C3] = f104, SIZE
	mov	f96  = f0
#if defined(TRMMKERNEL) && defined(LEFT)
	adds	KK = 1, KK
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C4] = f120, SIZE
	mov	f97 = f0
	nop	__LINE__
	}
	;;
	{ .mfi
	STFD	[C3] = f105, SIZE
	mov	f112 = f0
#ifdef TRMMKERNEL
	shladd	KK8 = KK, ZBASE_SHIFT, r0
#else
	nop	__LINE__
#endif
	}
	{ .mfi
	STFD	[C4] = f121, SIZE
	mov	f113 = f0
	nop	__LINE__
	}
	;;
#endif
	.align 16

.L049:
	{ .mmi
	mov	B = BOFFSET
	mov	AOFFSET = A
#if defined(TRMMKERNEL) && !defined(LEFT)
	adds	KK = 4, KK
#else
	nop	__LINE__
#endif
	}
	{ .mmb
	nop	__LINE__
	cmp.lt	p6, p0 = 0, J
	(p6)	br.cond.dptk .L010
	}
	;;
	.align 16

.L050:
	{ .mmi
#if defined(TRMMKERNEL) &&  defined(LEFT)
	mov	KK  = OFFSET
#else
	nop	__LINE__
#endif
	shr	I  = M, 2
	}
	{ .mib
	mov	C1 = C
	tbit.z	p6, p0 = N, 1
	(p6)	br.cond.dpnt .L090
	}
	;;
	{ .mmi
	add	C2 = LDC, C
#ifdef TRMMKERNEL
	shladd	KK8 = KK, ZBASE_SHIFT, r0
#else
	nop	__LINE__
#endif
	nop	__LINE__
	}
	{ .mib
	cmp.eq	p6, p7 = 0, I
	shladd	C = LDC, 1, C
	(p6)	br.cond.dpnt .L060
	}
	;;
	.align 16

.L052:
#if !defined(TRMMKERNEL) || \
    defined(TRMMKERNEL) && \
    ((defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)))
	{ .mfi
	LDFPD	f48, f49 = [B]
	mov	f66  = f0
	adds	PREA = (PREFETCHSIZE + 8) * SIZE, AOFFSET
	}
	{ .mfb
	adds	BOFFSET = 2 * SIZE, B
	mov	f67  = f0
	nop	__LINE__
	}
	;;
#else
	{ .mfi
	shladd	BOFFSET = KK8, 1, B
	mov	f66  = f0
	shladd	AOFFSET = KK8, 2, AOFFSET
	}
	;;
	{ .mfi
	LDFPD	f48, f49 = [BOFFSET], 2 * SIZE
	mov	f67  = f0
	adds	PREA = (PREFETCHSIZE + 8) * SIZE, AOFFSET
	}
	;;
#endif
	{ .mfi
	LDFPD	f32, f33 = [AOFFSET], 2 * SIZE
	mov	f82  = f0
	adds	PREC = CPREFETCHSIZE * SIZE, C1
	}
	{ .mfi
	LDFPD	f50, f51 = [BOFFSET], 2 * SIZE
	mov	f83  = f0
#ifndef TRMMKERNEL
	nop	__LINE__
#else
#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
	sub	L = K, KK
#elif defined(LEFT)
	adds	L = 4, KK
#else
	adds	L = 2, KK
#endif
#endif
	}
	;;
	{ .mfi
	LDFPD	f34, f35 = [AOFFSET], 2 * SIZE
	mov	f98  = f0
	adds	PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
	}
	{ .mfi
	cmp.eq	p3, p0 = r0, r0
	mov	f99  = f0
#ifndef TRMMKERNEL
	adds	L =  1, K
#else
	adds	L =  1, L
#endif
	}
	;;
	{ .mfi
	LDFPD	f36, f37 = [AOFFSET], 2 * SIZE
	mov	f114 = f0
	tbit.z	p12, p0 = L, 0
	}
	{ .mfi
	CPREFETCH [PREC], LDC
	mov	f115 = f0
	shr	L = L, 1
	}
	;;
	{ .mmi
	LDFPD	f38, f39 = [AOFFSET], 2 * SIZE
	adds C5 = 4 * SIZE, C1
	adds	L =  -1, L
	}
	;;
	{ .mmi
	CPREFETCH [PREC], LDC
	adds C6 = 4 * SIZE, C2
	mov	ar.lc = L
	}
	;;
	.align 16

.L053:
	{ .mfb
	lfetch.nt1	[PREA],  16 * SIZE
	FMA	f64   = f32, f48, f64	// A1 * B1
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	FMA_B	f65   = f32, f49, f65	// A1 * B2
	(p12) cmp.ne p3, p0 =  0, L
	}
	;;
	{ .mfi
	lfetch.nt1	[PREB],   8 * SIZE
	FMA	f80   = f32, f50, f80	// A1 * B3
	cmp.ne	p4, p5 =  0, L
	}
	{ .mfi
	nop	__LINE__
	FMA_B	f81   = f32, f51, f81	// A1 * B4
	nop	__LINE__
	}
	;;
	{ .mfi
	(p3) LDFPD	f40, f41 = [AOFFSET], 2 * SIZE
	FMA	f96   = f34, f48, f96	// A3 * B1
	nop	__LINE__
	}
	{ .mfi
	FMA_B	f97   = f34, f49, f97	// A3 * B2
	nop	__LINE__
	}
	;;
	{ .mfi
	(p3) LDFPD	f56, f57 = [BOFFSET],  2 * SIZE
	FMA	f112   = f34, f50, f112	// A3 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f113   = f34, f51, f113	// A3 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f58, f59 = [BOFFSET],  2 * SIZE
	FMA	f65   = f33, f48, f65	// A2 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f64   = f33, f49, f64	// A2 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f42, f43 = [AOFFSET], 2 * SIZE
	FMA	f81   = f33, f50, f81	// A2 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f80   = f33, f51, f80	// A2 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f44, f45 = [AOFFSET], 2 * SIZE
	FMA	f97   = f35, f48, f97	// A4 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f96   = f35, f49, f96	// A4 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p3) LDFPD	f46, f47 = [AOFFSET], 2 * SIZE
	FMA	f113   = f35, f50, f113	// A4 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f112   = f35, f51, f112	// A4 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f66   = f36, f48, f66	// A5 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f67   = f36, f49, f67	// A5 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f82   = f36, f50, f82	// A5 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f83   = f36, f51, f83	// A5 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f98   = f38, f48, f98	// A7 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f99   = f38, f49, f99	// A7 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f114   = f38, f50, f114	// A7 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_B	f115   = f38, f51, f115	// A7 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f67   = f37, f48, f67	// A6 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f66   = f37, f49, f66	// A6 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	nop	__LINE__
	FMA	f83   = f37, f50, f83	// A6 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f82   = f37, f51, f82	// A6 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f32, f33 = [AOFFSET],   2 * SIZE
	FMA	f99   = f39, f48, f99	// A8 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f98   = f39, f49, f98	// A8 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f48, f49 = [BOFFSET],  2 * SIZE
	FMA	f115   = f39, f50, f115	// A8 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FMA_A	f114   = f39, f51, f114	// A8 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f50, f51 = [BOFFSET],  2 * SIZE
	(p3) FMA	f64   = f40, f56, f64	// A1 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f65   = f40, f57, f65	// A1 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f34, f35 = [AOFFSET], 2 * SIZE
	(p3) FMA	f80   = f40, f58, f80	// A1 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f81   = f40, f59, f81	// A1 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f36, f37 = [AOFFSET], 2 * SIZE
	(p3) FMA	f96   = f42, f56, f96	// A3 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f97   = f42, f57, f97	// A3 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
	(p4) LDFPD	f38, f39 = [AOFFSET], 2 * SIZE
	(p3) FMA	f112   = f42, f58, f112	// A3 * B3
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA_B	f113   = f42, f59, f113	// A3 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f72  = [C1 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f65   = f41, f56, f65	// A2 * B1
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f76 = [C5 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_A	f64   = f41, f57, f64	// A2 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f73  = [C1 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f81   = f41, f58, f81	// A2 * B3
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f77 = [C5 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_A	f80   = f41, f59, f80	// A2 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f74 = [C1 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f97   = f43, f56, f97	// A4 * B1
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f78 = [C5 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_A	f96   = f43, f57, f96	// A4 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f75 = [C1 ], -3 * SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f113   = f43, f58, f113	// A4 * B3
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f79 = [C5 ], -3 * SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_A	f112   = f43, f59, f112	// A4 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f88 = [C2 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f66   = f44, f56, f66	// A5 * B1
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f92 = [C6 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_B	f67   = f44, f57, f67	// A5 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f89 = [C2 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f82   = f44, f58, f82	// A5 * B3
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f93 = [C6 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_B	f83   = f44, f59, f83	// A5 * B4
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f90 = [C2 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA	f98   = f46, f56, f98	// A7 * B1
	nop	__LINE__
	}
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f94 = [C6 ], SIZE
#else
	nop	__LINE__
#endif
	(p3) FMA_B	f99   = f46, f57, f99	// A7 * B2
	nop	__LINE__
	}
	;;
	{ .mfb
#if! defined(TRMMKERNEL) && !defined(BETAZERO)
	(p5) LDFD	f91 = [C2 ],