ztrsm_kernel_rt.s

Optimized GotoBLAS libraries

	;;
	FMPY	f32  = f104, f96
	FMPY	f33  = f105, f96
	FMPY	f34  = f104, f112
	FMPY	f35  = f105, f112
	;;
	FMA_C	f96  = f105, f97,  f32
	FMA_D	f97  = f104, f97,  f33
	FMA_C	f112 = f105, f113, f34
	FMA_D	f113 = f104, f113, f35
	;;
	FNMA	f64  = f106, f96,  f64
	FMA_A	f65  = f107, f96,  f65
	FNMA	f80  = f106, f112, f80
	FMA_A	f81  = f107, f112, f81
	;;
	FMA_B	f64  = f107, f97,  f64
	FNMA	f65  = f106, f97,  f65
	FMA_B	f80  = f107, f113, f80
	FNMA	f81  = f106, f113, f81
	;;
	FMPY	f32  = f120, f64
	FMPY	f33  = f121, f64
	FMPY	f34  = f120, f80
	FMPY	f35  = f121, f80
	;;
	FMA_C	f64  = f121, f65,  f32
	FMA_D	f65  = f120, f65,  f33
	FMA_C	f80  = f121, f81,  f34
	FMA_D	f81  = f120, f81,  f35
	;;
#endif

#ifdef LT
	LDFPD	f72, f73 = [AOFFSET], 2 * SIZE
	;;
	LDFPD	f74, f75 = [AOFFSET]
	adds	AOFFSET = 4 * SIZE, AOFFSET
	;;
	LDFPD	f90, f91 = [AOFFSET]
	adds	AOFFSET = - 6 * SIZE, AOFFSET
	;;
	FMPY	f32  = f72, f64
	FMPY	f33  = f73, f64
	FMPY	f34  = f72, f80
	FMPY	f35  = f73, f80
	;;
	FMA_C	f64  = f73, f65,  f32
	FMA_D	f65  = f72, f65,  f33
	FMA_C	f80  = f73, f81,  f34
	FMA_D	f81  = f72, f81,  f35
	;;
	FNMA	f96  = f74, f64, f96
	FMA_A	f97  = f75, f64, f97
	FNMA	f112 = f74, f80, f112
	FMA_A	f113 = f75, f80, f113
	;;
	FMA_B	f96  = f75, f65, f96
	FNMA	f97  = f74, f65, f97
	FMA_B	f112 = f75, f81, f112
	FNMA	f113 = f74, f81, f113
	;;
	FMPY	f32  = f90, f96
	FMPY	f33  = f91, f96
	FMPY	f34  = f90, f112
	FMPY	f35  = f91, f112
	;;
	FMA_C	f96  = f91, f97,  f32
	FMA_D	f97  = f90, f97,  f33
	FMA_C	f112 = f91, f113, f34
	FMA_D	f113 = f90, f113, f35
	;;
#endif

#ifdef RN
	LDFPD	f72, f73 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f74, f75 = [BOFFSET]
	adds	BOFFSET = 4 * SIZE, BOFFSET
	;;
	LDFPD	f90, f91 = [BOFFSET]
	adds	BOFFSET = - 6 * SIZE, BOFFSET
	;;
	FMPY	f32  = f72, f64
	FMPY	f33  = f73, f64
	FMPY	f34  = f72, f96
	FMPY	f35  = f73, f96
	;;
	FMA_C	f64  = f73, f65, f32
	FMA_D	f65  = f72, f65, f33
	FMA_C	f96  = f73, f97, f34
	FMA_D	f97  = f72, f97, f35
	;;
	FNMA	f80  = f74, f64, f80
	FMA_A	f81  = f75, f64, f81
	FNMA	f112 = f74, f96, f112
	FMA_A	f113 = f75, f96, f113
	;;
	FMA_B	f80  = f75, f65, f80
	FNMA	f81  = f74, f65, f81
	FMA_B	f112 = f75, f97, f112
	FNMA	f113 = f74, f97, f113

	;;
	FMPY	f32  = f90, f80
	FMPY	f33  = f91, f80
	FMPY	f34  = f90, f112
	FMPY	f35  = f91, f112
	;;
	FMA_C	f80  = f91, f81,  f32
	FMA_D	f81  = f90, f81,  f33
	FMA_C	f112 = f91, f113, f34
	FMA_D	f113 = f90, f113, f35
	;;
#endif

#ifdef RT
	adds	BOFFSET = 6 * SIZE, BOFFSET
	;;
	LDFPD	f104, f105 = [BOFFSET]
	adds	BOFFSET = - 2 * SIZE, BOFFSET
	;;
	LDFPD	f106, f107 = [BOFFSET]
	adds	BOFFSET = - 4 * SIZE, BOFFSET
	;;
	LDFPD	f120, f121 = [BOFFSET]
	;;
	FMPY	f32  = f104, f80
	FMPY	f33  = f105, f80
	FMPY	f34  = f104, f112
	FMPY	f35  = f105, f112
	;;
	FMA_C	f80  = f105, f81,  f32
	FMA_D	f81  = f104, f81,  f33
	FMA_C	f112 = f105, f113, f34
	FMA_D	f113 = f104, f113, f35
	;;
	FNMA	f64  = f106, f80,  f64
	FMA_A	f65  = f107, f80,  f65
	FNMA	f96  = f106, f112, f96
	FMA_A	f97  = f107, f112, f97
	;;
	FMA_B	f64  = f107, f81,  f64
	FNMA	f65  = f106, f81,  f65
	FMA_B	f96  = f107, f113, f96
	FNMA	f97  = f106, f113, f97
	;;
	FMPY	f32  = f120, f64
	FMPY	f33  = f121, f64
	FMPY	f34  = f120, f96
	FMPY	f35  = f121, f96
	;;
	FMA_C	f64  = f121, f65, f32
	FMA_D	f65  = f120, f65, f33
	FMA_C	f96  = f121, f97, f34
	FMA_D	f97  = f120, f97, f35
	;;
#endif

#if defined(LN) || defined(LT)
	adds	BOFFSET2 = 4 * SIZE, BOFFSET
	;;
	STFD	[BOFFSET]  = f64,  SIZE
	STFD	[BOFFSET2] = f96,  SIZE
	;;
	STFD	[BOFFSET]  = f65,  SIZE
	STFD	[BOFFSET2] = f97,  SIZE
	;;
	STFD	[BOFFSET]  = f80,  SIZE
	STFD	[BOFFSET2] = f112, SIZE
	;;
	STFD	[BOFFSET]  = f81,  5 * SIZE
	STFD	[BOFFSET2] = f113, 5 * SIZE
	;;
	adds	BOFFSET = - 8 * SIZE, BOFFSET
	;;
#else
	adds	AOFFSET2 = 4 * SIZE, AOFFSET
	;;
	STFD	[AOFFSET]  = f64,  SIZE
	STFD	[AOFFSET2] = f80,  SIZE
	;;
	STFD	[AOFFSET]  = f65,  SIZE
	STFD	[AOFFSET2] = f81,  SIZE
	;;
	STFD	[AOFFSET]  = f96,  SIZE
	STFD	[AOFFSET2] = f112, SIZE
	;;
	STFD	[AOFFSET]  = f97,  5 * SIZE
	STFD	[AOFFSET2] = f113, 5 * SIZE
	;;
	adds	AOFFSET = - 8 * SIZE, AOFFSET
	;;
#endif

#ifdef LN
	adds	C1 = -4 * SIZE, C1
	adds	C2 = -4 * SIZE, C2
#endif
	;;
	STFD	[C1 ] = f64, SIZE
	;;
	STFD	[C1 ] = f65, SIZE
	;;
	STFD	[C1 ] = f96, SIZE
	;;
	STFD	[C1 ] = f97, SIZE
	;;
	STFD	[C2 ] = f80, SIZE
	;;
	STFD	[C2 ] = f81, SIZE
	;;
	STFD	[C2 ] = f112, SIZE
	;;
	STFD	[C2 ] = f113, SIZE
	;;
	mov	f64 = f0
	mov	f65 = f0
	mov	f80 = f0
	mov	f81 = f0
	mov	f96 = f0
	mov	f97 = f0
	mov	f112 = f0
	mov	f113 = f0
	;;
#ifdef LN
	adds	C1 = -4 * SIZE, C1
	adds	C2 = -4 * SIZE, C2
#endif
	;;
	cmp.ne	p6, p0 = 1, I
	;;
	adds	I = -1, I
	;;
	shladd	r2 = K, ZBASE_SHIFT, r0
	;;
	sub	L = K, KK
	;;
#ifdef RT
	shladd	AORIG = r2, 1, AORIG
#endif
	;;
#if defined(LT) || defined(RN)
	shladd	L = L, ZBASE_SHIFT, r0
	;;
	shladd	AOFFSET = L, 1, AOFFSET
	shladd	BOFFSET = L, 1, BOFFSET
#endif
	;;
#ifdef LT
	adds	KK =  2, KK
#elif defined LN
	adds	KK = -2, KK
#else
	nop	__LINE__
#endif
	;;
#if defined(LT) || defined(RN)
	mov	L = KK
#else
	sub	L = K, KK
#endif
	;;
	.align 16

.L070:
	{ .mib
#if defined(LT) || defined(RN)
	mov	L = KK
#else
	sub	L = K, KK
#endif
	tbit.z	p6, p7 = M, 0
	(p6)	br.cond.dptk .L089
	}
	;;
	{ .mmi
	cmp.ne	p7, p0 = r0, L
	adds	BOFFSET = 0 * SIZE, B
	shl	r2 = K, ZBASE_SHIFT
	}
	{ .mmi
	shladd	r3 = KK, ZBASE_SHIFT, r0
	nop	__LINE__
	nop	__LINE__
	}
	;;
#if defined(LT) || defined(RN)
	{ .mfb
	(p7) LDFPD	f48, f49 = [BOFFSET], 2 * SIZE
	}
	;;
#else
	{ .mfi
	shladd	BOFFSET = r3, 1, B
#ifdef LN
	sub	AORIG = AORIG, r2
#else
	nop	__LINE__
#endif
	}
	;;
	{ .mfi
	(p7) LDFPD	f48, f49 = [BOFFSET], 2 * SIZE
	add	AOFFSET = r3, AORIG
	}
	;;
#endif
	;;
	adds	L =  1, L
	;;
	{ .mii
	(p7) LDFPD	f32, f33 = [AOFFSET], 2 * SIZE
	tbit.z	p12, p0 = L, 0
	shr	L = L, 1
	}
	;;
	{ .mmi
	(p7) LDFPD	f50, f51 = [BOFFSET], 2 * SIZE
	adds	PREA = (PREFETCHSIZE + 0) * SIZE, AOFFSET
	adds	L =  -1, L
	}
	;;
	{ .mmi
	adds	PREB = (PREFETCHSIZE + 0) * SIZE, BOFFSET
	cmp.eq	p3, p0 = r0, r0
	mov	ar.lc = L
	}
	;;
	cmp.eq  p6, p0 = -1, L
	(p6) br.cond.dpnt   .L078
	;;
	.align 16

.L072:
	{ .mfb
	lfetch.nt1	[PREA],  4 * SIZE
	FMA	f64   = f32, f48, f64	// A1 * B1
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	FMA	f96   = f32, f49, f96	// A1 * B2
	(p12) cmp.ne p3, p0 =  0, L
	}
	;;
	{ .mfi
	lfetch.nt1	[PREB],   8 * SIZE
	FMA	f80   = f32, f50, f80	// A1 * B3
	cmp.ne	p4, p5 =  0, L
	}
	{ .mfb
	nop	__LINE__
	FMA	f112   = f32, f51, f112	// A1 * B4
	nop	__LINE__
	}
	;;
	{ .mfi
	(p3) LDFPD	f40, f41 = [AOFFSET], 2 * SIZE
	FMA	f65   = f33, f48, f65	// A2 * B1
	}
	{ .mfi
	nop	__LINE__
	FMA	f97   = f33, f49, f97	// A2 * B2
	}
	;;
	{ .mfi
	(p3) LDFPD	f56, f57 = [BOFFSET], 2 * SIZE
	FMA	f81   = f33, f50, f81	// A2 * B3
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f113   = f33, f51, f113	// A2 * B4
	}
	;;
	{ .mfb
	(p3) LDFPD	f58, f59 = [BOFFSET], 2 * SIZE
	(p3) FMA	f64   = f40, f56, f64	// A1 * B1
	nop	__LINE__
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p3) FMA	f96   = f40, f57, f96	// A1 * B2
	}
	;;
	{ .mfb
	(p4) LDFPD	f32, f33 = [AOFFSET], 2 * SIZE
	(p3) FMA	f80   = f40, f58, f80	// A1 * B3
	nop	__LINE__
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p3) FMA	f112   = f40, f59, f112	// A1 * B4
	}
	;;
	{ .mfb
	(p4) LDFPD	f48, f49 = [BOFFSET],   2 * SIZE
	(p3) FMA	f65   = f41, f56, f65	// A2 * B1
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA	f97   = f41, f57, f97	// A2 * B2
	nop	__LINE__
	}
	;;
	{ .mfi
	(p4) LDFPD	f50, f51 = [BOFFSET],  2 * SIZE
	(p3) FMA	f81   = f41, f58, f81	// A2 * B3
	adds	L = -1, L
	}
	{ .mfb
	nop	__LINE__
	(p3) FMA	f113   = f41, f59, f113	// A2 * B4
	br.cloop.sptk.few .L072
	}
	;;
	{ .mfb
	nop	__LINE__
	FCALC_A	  f64  = f64,  f97
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_A	  f80  = f80,  f113
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_B	  f65  = f65,  f96
	nop	__LINE__
	}
	{ .mfb
	nop	__LINE__
	FCALC_B	  f81  = f81,  f112
	nop	__LINE__
	}
	;;
.L078:
#if defined(LN) || defined(RT)
#ifdef LN
	adds	r2 = -1, KK
#else
	adds	r2 = -2, KK
#endif
	;;
	shladd	r2 = r2, ZBASE_SHIFT, r0
	;;
	add	AOFFSET = r2, AORIG
	shladd	BOFFSET = r2, 1, B
	;;	
#endif

#if defined(LN) || defined(LT)
	LDFPD	f72, f73 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f74, f75 = [BOFFSET]
	adds	BOFFSET = -2 * SIZE, BOFFSET
	;;
	FSUB	f64  = f72, f64
	FSUB_A	f65  = f73, f65
	FSUB	f80  = f74, f80
	FSUB_A	f81  = f75, f81
	;;
#else
	LDFPD	f72, f73 = [AOFFSET], 2 * SIZE
	;;
	LDFPD	f88, f89 = [AOFFSET]
	adds	AOFFSET = -2 * SIZE, AOFFSET
	;;
	FSUB	f64  = f72, f64
	FSUB	f65  = f73, f65
	FSUB	f80  = f88, f80
	FSUB	f81  = f89, f81
	;;
#endif

#ifdef LN
	LDFPD	f120, f121 = [AOFFSET]
	;;
	FMPY	f32  = f120, f64
	FMPY	f33  = f121, f64
	FMPY	f34  = f120, f80
	FMPY	f35  = f121, f80
	;;
	FMA_C	f64  = f121, f65,  f32
	FMA_D	f65  = f120, f65,  f33
	FMA_C	f80  = f121, f81,  f34
	FMA_D	f81  = f120, f81,  f35
	;;
#endif

#ifdef LT
	LDFPD	f72, f73 = [AOFFSET]
	;;
	FMPY	f32  = f72, f64
	FMPY	f33  = f73, f64
	FMPY	f34  = f72, f80
	FMPY	f35  = f73, f80
	;;
	FMA_C	f64  = f73, f65,  f32
	FMA_D	f65  = f72, f65,  f33
	FMA_C	f80  = f73, f81,  f34
	FMA_D	f81  = f72, f81,  f35
	;;
#endif

#ifdef RN
	LDFPD	f72, f73 = [BOFFSET], 2 * SIZE
	;;
	LDFPD	f74, f75 = [BOFFSET]
	adds	BOFFSET = 4 * SIZE, BOFFSET
	;;
	LDFPD	f90, f91 = [BOFFSET]
	adds	BOFFSET = - 6 * SIZE, BOFFSET
	;;
	FMPY	f32  = f72, f64
	FMPY	f33  = f73, f64
	;;
	FMA_C	f64  = f73, f65, f32
	FMA_D	f65  = f72, f65, f33
	;;
	FNMA	f80  = f74, f64, f80
	FMA_A	f81  = f75, f64, f81
	;;
	FMA_B	f80  = f75, f65, f80
	FNMA	f81  = f74, f65, f81

	;;
	FMPY	f32  = f90, f80
	FMPY	f33  = f91, f80
	;;
	FMA_C	f80  = f91, f81,  f32
	FMA_D	f81  = f90, f81,  f33
	;;
#endif

#ifdef RT
	adds	BOFFSET = 6 * SIZE, BOFFSET
	;;
	LDFPD	f104, f105 = [BOFFSET]
	adds	BOFFSET = - 2 * SIZE, BOFFSET
	;;
	LDFPD	f106, f107 = [BOFFSET]
	adds	BOFFSET = - 4 * SIZE, BOFFSET
	;;
	LDFPD	f120, f121 = [BOFFSET]
	;;
	FMPY	f32  = f104, f80
	FMPY	f33  = f105, f80
	;;
	FMA_C	f80  = f105, f81,  f32
	FMA_D	f81  = f104, f81,  f33
	;;
	FNMA	f64  = f106, f80,  f64
	FMA_A	f65  = f107, f80,  f65
	;;
	FMA_B	f64  = f107, f81,  f64
	FNMA	f65  = f106, f81,  f65
	;;
	FMPY	f32  = f120, f64
	FMPY	f33  = f121, f64
	;;
	FMA_C	f64  = f121, f65, f32
	FMA_D	f65  = f120, f65, f33
	;;
#endif

#if defined(LN) || defined(LT)
	STFD	[BOFFSET]  = f64,  SIZE
	;;
	STFD	[BOFFSET]  = f65,  SIZE
	;;
	STFD	[BOFFSET]  = f80,  SIZE
	;;
	STFD	[BOFFSET]  = f81,  SIZE
	;;
	adds	BOFFSET = - 4 * SIZE, BOFFSET
	;;
#else
	STFD	[AOFFSET]  = f64,  SIZE
	;;
	STFD	[AOFFSET]  = f65,  SIZE
	;;
	STFD	[AOFFSET]  = f80,  SIZE
	;;
	STFD	[AOFFSET]  = f81,  SIZE
	;;
	adds	AOFFSET = - 4 * SIZE, AOFFSET
	;;
#endif

#ifdef LN
	adds	C1 = -2 * SIZE, C1
	adds	C2 = -2 * SIZE, C2
#endif
	;;
	STFD	[C1 ] = f64, SIZE
	;;
	STFD	[C1 ] = f65, SIZE
	;;
	STFD	[C2 ] = f80, SIZE
	;;
	STFD	[C2 ] = f81, SIZE
	;;
	mov	f64 = f0
	mov	f65 = f0
	mov	f80 = f0
	mov	f81 = f0
	mov	f96 = f0
	mov	f97 = f0
	mov	f112 = f0
	mov	f113 = f0
	;;
#ifdef LN
	adds	C1 = -2 * SIZE, C1
	adds	C2 = -2 * SIZE, C2
#endif
	;;
	cmp.ne	p6, p0 = 1, I
	;;
	adds	I = -1, I
	;;
	shladd	r2 = K, ZBASE_SHIFT, r0
	;;
	sub	L = K, KK
	;;
#ifdef RT
	add	AORIG = r2, AORIG
#endif
	;;
#if defined(LT) || defined(RN)
	shladd	L = L, ZBASE_SHIFT, r0
	;;
	add	AOFFSET = L, AOFFSET
	shladd	BOFFSET = L, 1, BOFFSET
#endif
	;;
#ifdef LT
	adds	KK =  1, KK
#elif defined LN
	adds	KK = -1, KK
#else
	nop	__LINE__
#endif
	;;
#if defined(LT) || defined(RN)
	mov	L = KK
#else
	sub	L = K, KK
#endif
	;;
	.align 16

.L089:
#ifdef LN
	shladd	KK8 = K, ZBASE_SHIFT, r0
	;;
       shladd	B = KK8, 1, B
#endif

#if defined(LT) || defined(RN)
	mov	B =  BOFFSET
#endif

#ifdef RN
	adds	KK =  2,  KK
#endif

#ifdef RT
	adds	KK = -2,  KK
#endif
	;;
	{ .mmi
	mov	AOFFSET = A
	nop	__LINE__
	}
	;;
	.align 16

.L010:
	shr	J = N, 2
	;;
	cmp.ge	p6, p0  = 0, J
	(p6)	br.cond.dpnt .L999
	;;

.L010x:
#ifdef RT
       { .mmi
	shladd	r3 = LDC, 2, r0
	nop	__LINE__
	shl	r2 = K, 2 + ZBASE_SHIFT
	}
	;;
	{ .mmi
	sub	B = B, r2
	sub	C = C, r3
	nop	__LINE__
	}
	;;
#endif
	{ .mmi
	mov	C1 = C			// coffset1 = c + 0 * ldc
	add	C2 = LDC, C		// coffset2 = c + 1 * ldc
	shr	I  = M, 2
	}
	{ .mmi
	adds	J = -1, J
#ifdef LN
	add	KK = M, OFFSET
#elif defined LT
	mov	KK = OFFSET
#else
	nop	__LINE__
#endif
#if defined(LN) || defined(RT)
	mov	AORIG = A
#else
	mov	AOFFSET = A
#endif
	}
	;;
	;;
	{ .mmi
	shladd	C3 = LDC, 1, C		// coffset3 = c + 2 * ldc
	shladd	C4 = LDC, 1, C2		// coffset4 = c + 3 * ldc
#if defined(LT) || defined(RN)
	mov	L = KK
#else
	sub	L = K, KK
#endif
	}
	{ .mib
	cmp.eq	p6, p7 = 0, I
#ifndef RT
	shladd	C = LDC, 2, C		// coffset += 8 * ldc
#else
	nop	__LINE__
#endif
	(p6)	br.cond.dpnt .L020
	}
	;;
	.align 16

.L011:
	{ .mmi
	cmp.ne	p7, p0 = r0, L
	adds	BOFFSET = 0 * SIZE, B
	shl	r2 = K, 2 + ZBASE_SHIFT
	}
	{ .mfi
	shladd	r3 = KK, ZBASE_SHIFT, r0
	mov	f118 = f0
	nop	__LINE__
	}
	;;
#if