gemv_t.s

Optimized GotoBLAS libraries

	.align 16

.L45:
	and	I = 15, MIN_M
	;;
	cmp.eq	p6,  p0 = 0, I
	(p6) br.cond.dpnt .L48
	;;
	tbit.nz	p12, p0 = MIN_M, 3
	tbit.nz	p13, p0 = MIN_M, 2
	tbit.nz	p14, p0 = MIN_M, 1
	tbit.nz	p15, p0 = MIN_M, 0
	;;
	(p12) LDFPD	f32,  f33  = [AO1], 2 * SIZE
	(p12) LDFPD	f100, f101 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f36,  f37  = [AO1], 2 * SIZE
	(p12) LDFPD	f102, f103 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f40,  f41  = [AO1], 2 * SIZE
	(p12) LDFPD	f104, f105 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f44,  f45  = [AO1], 2 * SIZE
	(p12) LDFPD	f106, f107 = [BO], 2 * SIZE
	;;
	(p13) LDFPD	f48,  f49  = [AO1], 2 * SIZE
	(p13) LDFPD	f108, f109 = [BO], 2 * SIZE
	;;
	(p13) LDFPD	f52,  f53  = [AO1], 2 * SIZE
	(p13) LDFPD	f110, f111 = [BO], 2 * SIZE
	;;
	(p14) LDFPD	f56,  f57  = [AO1], 2 * SIZE
	(p14) LDFPD	f112, f113 = [BO], 2 * SIZE
	;;
	(p15) LDFD	f60        = [AO1]
	(p15) LDFD	f114       = [BO]
	;;
	(p12) FMA	f8  = f100, f32, f8
	(p12) FMA	f9  = f101, f33, f9
	(p12) FMA	f10 = f102, f36, f10
	(p12) FMA	f11 = f103, f37, f11
	(p12) FMA	f12 = f104, f40, f12
	(p12) FMA	f13 = f105, f41, f13
	(p12) FMA	f14 = f106, f44, f14
	(p12) FMA	f15 = f107, f45, f15
	;;
	(p13) FMA	f8  = f108, f48, f8
	(p13) FMA	f9  = f109, f49, f9
	(p13) FMA	f10 = f110, f52, f10
	(p13) FMA	f11 = f111, f53, f11
	(p14) FMA	f12 = f112, f56, f12
	(p14) FMA	f13 = f113, f57, f13
	(p15) FMA	f14 = f114, f60, f14
	;;
.L48:
	{ .mmf
	LDFD	f32 = [CO]
	nop	__LINE__
	FADD	f8  = f8,  f9
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f10 = f10, f11
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f12 = f12, f13
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f14 = f14, f15
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f8  = f8,  f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f10 = f10, f14
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f8  = f8,  f10
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f32 = ALPHA, f8,  f32
	}
	;;
	{ .mmf
	STFD [CO] = f32
	}
	;;
	.align 16

.L99:
	adds	IS = P, IS
	shladd	A  = LDAP, BASE_SHIFT, A
	;;
	cmp.gt	p6, p0 = M, IS
	(p6) br.cond.dptk .LIs_loop
	br   .L999
	.align 4
	;;

.L100:
	shr	J   = N, 3
	mov	CO  = Y
	;;
	cmp.eq	p6, p0 = r0, J
	(p6) br.cond.dpnt .L120
	;;
	.align 16

.L111:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	shr	I = MIN_M, 4
	}
	;;
	{ .mmf
	shladd	AO3 = LDA, 1, A
	shladd	AO4 = LDA, 1, AO2
	mov	f12 = f0
	}
	{ .mmf
	(p8) LDFD f32 = [AO1], SIZE
	(p8) LDFD f33 = [AO2], SIZE
	mov	f14 = f0
	}
	;;
	{ .mmf
	shladd	AO5 = LDA, 1, AO3
	shladd	AO6 = LDA, 1, AO4
	mov	f16 = f0
	}
	{ .mmf
	(p8) LDFD f34 = [AO3], SIZE
	(p8) LDFD f35 = [AO4], SIZE
	mov	f18 = f0
	}
	;;
	{ .mmf
	shladd	AO7 = LDA, 1, AO5
	shladd	AO8 = LDA, 1, AO6
	mov	f20 = f0
	}
	{ .mmf
	(p8) LDFD f36 = [AO5], SIZE
	(p8) LDFD f37 = [AO6], SIZE
	mov	f22 = f0
	}
	;;
	{ .mfi
	(p8) LDFD f38 = [AO7], SIZE
	mov	f9  = f0
	mov	ar.ec= 2
	}
	{ .mmf
	(p8) LDFD f39 = [AO8], SIZE
	mov	BO  = BUFFER
	mov	f11 = f0
	}
	;;
	{ .mmf
	(p8) LDFD f40 = [BO], 2 * SIZE
	cmp.eq	p6, p0 = 0, I
	mov	f13 = f0
	}
	{ .mmf
	shladd	A   = LDA, 3, A
	cmp.eq	p16, p0 = r0, r0
	mov	f15 = f0
	}
	;;
	{ .mmf
	add	I = I, I
	nop	__LINE__
	mov	f17 = f0
	}
	{ .mmf
	adds	RPRE1  = RPREFETCH * SIZE, AO1
	adds	RPRE2  = (RPREFETCH + 8) * SIZE, AO2
	mov	f19 = f0
	}
	;;
	{ .mmf
	adds	I = -1, I
	nop	__LINE__
	mov	f21 = f0
	}
	{ .mmf
	adds	RPRE3  = RPREFETCH * SIZE, AO3
	adds	RPRE4  = (RPREFETCH + 8) * SIZE, AO4
	mov	f23 = f0
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p8) FMPY	f8  = f40, f32
	}
	{ .mmf
	adds	RPRE5  = RPREFETCH * SIZE, AO5
	adds	RPRE6  = (RPREFETCH + 8) * SIZE, AO6
	(p8) FMPY	f10 = f40, f33
	}
	;;
	{ .mmf
	adds	AO21 = 7 * SIZE, AO2
	adds	AO41 = 7 * SIZE, AO4
	(p8) FMPY	f12 = f40, f34
	}
	{ .mmf
	adds	RPRE7  = RPREFETCH * SIZE, AO7
	adds	RPRE8  = (RPREFETCH + 8) * SIZE, AO8
	(p8) FMPY	f14 = f40, f35
	}
	;;
	{ .mfi
	nop	__LINE__
	(p8) FMPY	f16 = f40, f36
	mov	ar.lc = I
	}
	{ .mmf
	adds	WPRE = 8 * SIZE, CO
	adds	PREB  = RPREFETCH * SIZE, BO
	(p8) FMPY	f18 = f40, f37
	}
	;;
	{ .mmf
	lfetch.excl.nt1	[WPRE]
	adds	AO61 = 7 * SIZE, AO6
	(p8) FMPY	f20 = f40, f38
	}
	{ .mfb
	adds	AO81 = 7 * SIZE, AO8
	(p8) FMPY	f22 = f40, f39
	(p6) br.cond.dpnt .L115
	}
	;;
	.align 16

.L112:
	{ .mmf
	(p17) LDFPD	f80, f95 = [AO8]
	(p17) LDFPD	f110, f111 = [BO], 2 * SIZE
	(p17) FMA	f8  = f104, f33, f8
	}
	{ .mfi
	(p17) adds	AO8 = 3 * SIZE, AO8
	(p17) FMA	f9  = f105, f34, f9
	(p16) tbit.nz.unc p14, p15 = I, 0
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFPD	f32, f33 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f104, f35, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f105, f36, f11
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE2], 16 * SIZE
	(p16) LDFD	f34      = [AO2], 1 * SIZE
	(p17) FMA	f12 = f104, f37, f12
	}
	{ .mmf
	(p17) LDFD	f84      = [AO21], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f13 = f105, f38, f13
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE3], 16 * SIZE
	(p16) LDFPD	f36, f37 = [AO3], 2 * SIZE
	(p17) FMA	f14 = f104, f39, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f105, f40, f15
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE4], 16 * SIZE
	(p16) LDFD	f38      = [AO4], 1 * SIZE
	(p17) FMA	f16 = f104, f41, f16
	}
	{ .mmf
	(p17) LDFD	f88      = [AO41], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f17 = f105, f42, f17
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE5], 16 * SIZE
	(p16) LDFPD	f40, f41 = [AO5], 2 * SIZE
	(p17) FMA	f18 = f104, f43, f18
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f19 = f105, f44, f19
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE6], 16 * SIZE
	(p16) LDFD	f42      = [AO6], 1 * SIZE
	(p17) FMA	f20 = f104, f45, f20
	}
	{ .mmf
	(p17) LDFD	f92      = [AO61], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f21 = f105, f46, f21
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE7], 16 * SIZE
	(p16) LDFPD	f44, f45 = [AO7], 2 * SIZE
	(p17) FMA	f22 = f104, f47, f22
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f23 = f105, f48, f23
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE8], 16 * SIZE
	(p16) LDFD	f46      = [AO8], 1 * SIZE
	(p17) FMA	f8  = f106, f49, f8
	}
	{ .mmf
	(p17) LDFD	f96      = [AO81], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f9  = f107, f50, f9
	}
	;;
	{ .mmf
	(p14) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f48, f49 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f106, f51, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f107, f52, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f35, f50 = [AO2], 2 * SIZE
	(p16) LDFPD	f103, f104 = [BO], 2 * SIZE
	(p17) FMA	f12 = f106, f53, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f13 = f107, f54, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f52, f53 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f106, f55, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f107, f56, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f39, f54 = [AO4], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f16 = f106, f57, f16
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f17 = f107, f58, f17
	}
	;;
	{ .mmf
	(p16) LDFPD	f56, f57 = [AO5], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f18 = f106, f59, f18
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f19 = f107, f60, f19
	}
	;;
	{ .mmf
	(p16) LDFPD	f43, f58 = [AO6], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f20 = f106, f61, f20
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f21 = f107, f62, f21
	}
	;;
	{ .mmf
	(p16) LDFPD	f60, f61 = [AO7], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f22 = f106, f63, f22
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f23 = f107, f64, f23
	}
	;;
	{ .mmf
	(p16) LDFPD	f47, f62 = [AO8], 2 * SIZE
	(p16) LDFPD	f105, f106 = [BO], 2 * SIZE
	(p17) FMA	f8  = f108, f65, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f109, f66, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f64, f65 = [AO1], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f10 = f108, f67, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f109, f68, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f51, f66 = [AO2], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f12 = f108, f69, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f13 = f109, f70, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f68, f69 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f108, f71, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f109, f72, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f55, f70 = [AO4], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f16 = f108, f73, f16
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f17 = f109, f74, f17
	}
	;;
	{ .mmf
	(p16) LDFPD	f72, f73 = [AO5], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f18 = f108, f75, f18
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f19 = f109, f76, f19
	}
	;;
	{ .mmf
	(p16) LDFPD	f59, f74 = [AO6], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f20 = f108, f77, f20
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f21 = f109, f78, f21
	}
	;;
	{ .mmf
	(p16) LDFPD	f76, f77 = [AO7], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f22 = f108, f79, f22
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f23 = f109, f80, f23
	}
	;;
	{ .mmf
	(p16) LDFPD	f63, f78 = [AO8], 2 * SIZE
	(p16) LDFPD	f107, f108 = [BO], 2 * SIZE
	(p17) FMA	f8  = f110, f81, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f111, f82, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f80, f81 = [AO1], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f10 = f110, f83, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f111, f84, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f67, f82 = [AO2]
	nop	__LINE__
	(p17) FMA	f12 = f110, f85, f12
	}
	{ .mmf
	nop	__LINE__
	(p16) adds	AO2 = 3 * SIZE, AO2
	(p17) FMA	f13 = f111, f86, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f84, f85 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f110, f87, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f111, f88, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f71, f86 = [AO4]
	nop	__LINE__
	(p17) FMA	f16 = f110, f89, f16
	}
	{ .mmf
	nop	__LINE__
	(p16) adds	AO4 = 3 * SIZE, AO4
	(p17) FMA	f17 = f111, f90, f17
	}
	;;
	{ .mmf
	(p16) LDFPD	f88, f89 = [AO5], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f18 = f110, f91, f18
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f19 = f111, f92, f19
	}
	;;
	{ .mmf
	(p16) LDFPD	f75, f90 = [AO6]
	nop	__LINE__
	(p17) FMA	f20 = f110, f93, f20
	}
	{ .mmf
	nop	__LINE__
	(p16) adds	AO6 = 3 * SIZE, AO6
	(p17) FMA	f21 = f111, f94, f21
	}
	;;
	{ .mmf
	(p16) LDFPD	f92, f93 = [AO7], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f22 = f110, f95, f22
	}
	{ .mfb
	adds	I = -1, I
	(p17) FMA	f23 = f111, f96, f23
	br.ctop.sptk.few .L112
	}
	;;
	.align 16

.L115:
	and	I = 15, MIN_M
	mov	pr.rot= 0
	;;
	cmp.eq	p6,  p0 = 0, I
	cmp.eq	p16, p15 = r0, r0
	;;
	adds	I = 1, I
	;;
	shr	I = I, 1
	;;
	adds	I = -1, I
	adds	AO21 = 1 * SIZE, AO2
	adds	AO41 = 1 * SIZE, AO4
	adds	AO61 = 1 * SIZE, AO6
	adds	AO81 = 1 * SIZE, AO8
	;;
	mov	ar.lc = I
	mov	ar.ec= 3
	and	I = 15, MIN_M
	(p6) br.cond.dpnt .L118
	;;
	.align 16

.L116:
	{ .mmf
	(p16) LDFPD	f104, f107 = [BO], 2 * SIZE
	(p16) LDFPD	f32,  f35  = [AO1], 2 * SIZE
	(p18) FMA	f8  = f106, f34, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f9  = f109, f37, f9
	}
	;;
	{ .mmf
	(p16) LDFD	f38  = [AO2], 2 * SIZE
	(p17) LDFD	f42  = [AO21], 2 * SIZE
	(p18) FMA	f10 = f106, f40, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f11 = f109, f43, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f44,  f47  = [AO3], 2 * SIZE
	nop	__LINE__
	(p18) FMA	f12 = f106, f46, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f13 = f109, f49, f13
	}
	;;
	{ .mmf
	(p16) LDFD	f50  = [AO4], 2 * SIZE
	(p17) LDFD	f54  = [AO41], 2 * SIZE
	(p18) FMA	f14 = f106, f52, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f15 = f109, f55, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f56,  f59  = [AO5], 2 * SIZE
	nop	__LINE__
	(p18) FMA	f16 = f106, f58, f16
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f17 = f109, f61, f17
	}
	;;
	{ .mmf
	(p16) LDFD	f62  = [AO6], 2 * SIZE
	(p17) LDFD	f66  = [AO61], 2 * SIZE
	(p18) FMA	f18 = f106, f64, f18
	}
	{ .mmf
	nop	__LINE__
	(p17) adds	I = -2, I
	(p15) FMA	f19 = f109, f67, f19
	}
	;;
	{ .mmf
	(p16) LDFPD	f68,  f71  = [AO7], 2 * SIZE
	nop	__LINE__
	(p18) FMA	f20 = f106, f70, f20
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f21 = f109, f73, f21
	}
	;;
	{ .mmf
	(p16) LDFD	f74  = [AO8], 2 * SIZE
	(p17) LDFD	f78  = [AO81], 2 * SIZE
	(p15) FMA	f23 = f109, f79, f23
	}
	{ .mfb
	(p17) cmp.ne.unc p15, p0 = -1, I
	(p18) FMA	f22 = f106, f76, f22
	br.ctop.sptk.few .L116
	}
	;;

.L118:
	{ .mmf
	mov	AO1 = CO
	LDFD	f32 = [CO], INCY
	FADD	f8  = f8,  f9
	}
	;;
	{ .mmf
	LDFD	f33 = [CO], INCY
	nop	__LINE__
	FADD	f10 = f10, f11
	}
	;;
	{ .mmf
	LDFD	f34 = [CO], INCY
	nop	__LINE__
	FADD	f12 = f12, f13
	}
	;;
	{ .mmf
	LDFD	f35 = [CO], INCY
	nop	__LINE__
	FADD	f14 = f14, f15
	}
	;;
	{ .mmf
	LDFD	f36 = [CO], INCY
	nop	__LINE__
	FADD	f16 = f16, f17
	}
	;;
	{ .mmf
	LDFD	f37 = [CO], INCY
	nop	__LINE__
	FADD	f18 = f18, f19
	}
	;;
	{ .mmf
	LDFD	f38 = [CO], INCY
	nop	__LINE__
	FADD	f20 = f20, f21
	}
	;;
	{ .mmf
	LDFD	f39 = [CO], INCY
	nop	__LINE__
	FADD	f22 = f22, f23
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f32 = ALPHA, f8,  f32
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f33 = ALPHA, f10, f33
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f34 = ALPHA, f12, f34
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f35 = ALPHA, f14, f35
	}
	;;
	{ .mmf
	STFD [AO1] = f32
	add	AO1 = AO1, INCY
	FMA	f36 = ALPHA, f16, f36
	}
	;;
	{ .mmf
	STFD [AO1] = f33
	add	AO1 = AO1, INCY
	FMA	f37 = ALPHA, f18, f37
	}
	;;
	{ .mmf
	STFD [AO1] = f34
	add	AO1 = AO1, INCY
	FMA	f38 = ALPHA, f20, f38
	}
	;;
	{ .mmf
	STFD [AO1] = f35
	add	AO1 = AO1, INCY
	FMA	f39 = ALPHA, f22, f39
	}
	;;
	{ .mmi
	STFD [AO1] = f36
	add	AO1 = AO1, INCY
	adds J = -1, J
	}
	;;
	{ .mmi
	STFD [AO1] = f37
	add	AO1 = AO1, INCY
	nop	__LINE__
	}
	;;
	{ .mmi
	STFD [AO1] = f38
	add	AO1 = AO1, INCY
	cmp4.lt p6, p0 = 0, J
	}
	;;
	{ .mib
	STFD [AO1] = f39
	add	AO1 = AO1, INCY
	(p6) br.cond.dptk .L111
	}
	;;
	.align 16

.L120:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	tbit.z	p6, p0  = N, 2
	}
	;;
	{ .mfi
	shladd	AO3 = LDA, 1, A
	mov	f12 = f0
	shr	I = MIN_M, 4
	}
	{ .mfb
	shladd	AO4 = LDA, 1, AO2
	mov	f14 = f0
	(p6) br.cond.dpnt .L130
	}
	;;
	{ .mmf
	(p8) LDFD f32 = [AO1], SIZE
	(p8) LDFD f33 = [AO2], SIZE
	mov	f9  = f0
	}
	{ .mmf
	mov	BO  = BUFFER
	shladd	A   = LDA, 2, A
	mov	f11 = f0
	}
	;;
	{ .mmf
	(p8) LDFD f40 = [BO], 2 * SIZE
	cmp.eq	p6, p0 = 0, I