zgemm3m_kernel_4x4_sse2.s
来自「Optimized GotoBLAS libraries」· S 代码 · 共 2,789 行 · 第 1/4 页
S
2,789 行
movsd 44 * SIZE(BO), %xmm3 mulsd %xmm0, %xmm3 mulsd 46 * SIZE(BO), %xmm0 addsd %xmm3, %xmm10 movsd 72 * SIZE(BO), %xmm3 addsd %xmm0, %xmm11 movsd -10 * SIZE(AO), %xmm0 mulsd %xmm0, %xmm5 addsd %xmm5, %xmm8 movsd 50 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm5 addsd %xmm5, %xmm9 movsd 52 * SIZE(BO), %xmm5 mulsd %xmm0, %xmm5 mulsd 54 * SIZE(BO), %xmm0 addsd %xmm5, %xmm10 movsd 80 * SIZE(BO), %xmm5 addsd %xmm0, %xmm11 movsd -9 * SIZE(AO), %xmm0 mulsd %xmm0, %xmm7 addsd %xmm7, %xmm8 movsd 58 * SIZE(BO), %xmm7 mulsd %xmm0, %xmm7 addsd %xmm7, %xmm9 movsd 60 * SIZE(BO), %xmm7 mulsd %xmm0, %xmm7 mulsd 62 * SIZE(BO), %xmm0 addsd %xmm7, %xmm10 movsd 88 * SIZE(BO), %xmm7 addsd %xmm0, %xmm11 movsd -8 * SIZE(AO), %xmm0 addq $ 8 * SIZE, AO addq $64 * SIZE, BO decq %rax jne .L32 ALIGN_3.L35:#ifndef TRMMKERNEL movq K, %rax#else movq KKK, %rax#endif movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH je .L38 ALIGN_3.L36: mulsd %xmm0, %xmm1 addsd %xmm1, %xmm8 movsd 2 * SIZE(BO), %xmm1 mulsd %xmm0, %xmm1 addsd %xmm1, %xmm9 movsd 4 * SIZE(BO), %xmm1 mulsd %xmm0, %xmm1 mulsd 6 * SIZE(BO), %xmm0 addsd %xmm1, %xmm10 movsd 8 * SIZE(BO), %xmm1 addsd %xmm0, %xmm11 movsd -15 * SIZE(AO), %xmm0 addq $1 * SIZE, AO # aoffset += 4 addq $8 * SIZE, BO # boffset1 += 8 decq %rax jg .L36 ALIGN_3.L38: movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movddup %xmm8, %xmm4 mulpd %xmm7, %xmm4 addpd %xmm4, %xmm0 movsd %xmm0, 0 * SIZE(CO1) movhpd %xmm0, 1 * SIZE(CO1) movsd 0 * SIZE(CO2), %xmm0 movhpd 1 * SIZE(CO2), %xmm0 movddup %xmm9, %xmm4 mulpd %xmm7, %xmm4 addpd %xmm4, %xmm0 movsd %xmm0, 0 * SIZE(CO2) movhpd %xmm0, 1 * SIZE(CO2) movsd 0 * SIZE(CO1, LDC, 2), %xmm0 movhpd 1 * SIZE(CO1, LDC, 2), %xmm0 movddup %xmm10, %xmm4 mulpd %xmm7, %xmm4 addpd %xmm4, %xmm0 movsd %xmm0, 0 * SIZE(CO1, LDC, 2) movhpd %xmm0, 1 * SIZE(CO1, LDC, 2) movsd 0 * SIZE(CO2, LDC, 2), %xmm0 movhpd 1 * SIZE(CO2, LDC, 2), %xmm0 movddup %xmm11, %xmm4 mulpd %xmm7, %xmm4 addpd %xmm4, %xmm0 movsd %xmm0, 0 * SIZE(CO2, LDC, 2) movhpd %xmm0, 1 * SIZE(CO2, LDC, 2) ALIGN_3 .L39:#if defined(TRMMKERNEL) && !defined(LEFT) addl $4, KK#endif leaq (C, LDC, 4), C # c += 4 * ldc decq J # j -- jg .L01 ALIGN_3.L40: testq $3, N je .L999 testq $2, N je .L80 ALIGN_4.L41:/* Copying to Sub Buffer */ leaq BUFFER, BO #if defined(TRMMKERNEL) && defined(LEFT) movq OFFSET, %rax movq %rax, KK#endif movq K, %rax sarq $2, %rax jle .L43 ALIGN_3 .L42: PREFETCH 56 * SIZE(B) movq 0 * SIZE(B), %mm0 movq 1 * SIZE(B), %mm1 movq 2 * SIZE(B), %mm2 movq 3 * SIZE(B), %mm3 movq 4 * SIZE(B), %mm4 movq 5 * SIZE(B), %mm5 movq 6 * SIZE(B), %mm6 movq 7 * SIZE(B), %mm7 addq $ 8 * SIZE, B addq $16 * SIZE, BO movq %mm0, -16 * SIZE(BO) movq %mm0, -15 * SIZE(BO) movq %mm1, -14 * SIZE(BO) movq %mm1, -13 * SIZE(BO) movq %mm2, -12 * SIZE(BO) movq %mm2, -11 * SIZE(BO) movq %mm3, -10 * SIZE(BO) movq %mm3, -9 * SIZE(BO) movq %mm4, -8 * SIZE(BO) movq %mm4, -7 * SIZE(BO) movq %mm5, -6 * SIZE(BO) movq %mm5, -5 * SIZE(BO) movq %mm6, -4 * SIZE(BO) movq %mm6, -3 * SIZE(BO) movq %mm7, -2 * SIZE(BO) movq %mm7, -1 * SIZE(BO) decq %rax jne .L42 ALIGN_3.L43: movq K, %rax andq $3, %rax BRANCH jle .L50 ALIGN_3.L44: movq 0 * SIZE(B), %mm0 movq 1 * SIZE(B), %mm1 movq %mm0, 0 * SIZE(BO) movq %mm0, 1 * SIZE(BO) movq %mm1, 2 * SIZE(BO) movq %mm1, 3 * SIZE(BO) addq $2 * SIZE, B addq $4 * SIZE, BO decq %rax jne .L44 ALIGN_3 .L50: movq C, CO1 # coffset1 = c leaq (C, LDC, 1), CO2 # coffset2 = c + ldc movq A, AO # aoffset = a movq M, I sarq $2, I # i = (m >> 2) jle .L60 ALIGN_3.L51:#if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER, BO#else leaq BUFFER, BO movq KK, %rax leaq (, %rax, SIZE), %rax leaq (AO, %rax, 4), AO leaq (BO, %rax, 4), BO#endif movapd -16 * SIZE(AO), %xmm0 pxor %xmm8, %xmm8 movapd 0 * SIZE(BO), %xmm1 pxor %xmm9, %xmm9 movapd -8 * SIZE(AO), %xmm2 pxor %xmm12, %xmm12 movapd 8 * SIZE(BO), %xmm3 pxor %xmm13, %xmm13 movapd 0 * SIZE(AO), %xmm4 movapd 16 * SIZE(BO), %xmm5 movapd 8 * SIZE(AO), %xmm6 movapd 24 * SIZE(BO), %xmm7 PREFETCHW 7 * SIZE(CO1) PREFETCHW 7 * SIZE(CO2)#ifndef TRMMKERNEL movq K, %rax#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) movq K, %rax subq KK, %rax movq %rax, KKK #else movq KK, %rax#ifdef LEFT addq $4, %rax#else addq $2, %rax#endif movq %rax, KKK#endif sarq $3, %rax je .L55 ALIGN_3.L52: mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 movapd 0 * SIZE(BO), %xmm1 addpd %xmm0, %xmm9 movapd -14 * SIZE(AO), %xmm0 mulpd %xmm0, %xmm1 mulpd 2 * SIZE(BO), %xmm0 addpd %xmm1, %xmm12 movapd 4 * SIZE(BO), %xmm1 addpd %xmm0, %xmm13 movapd -12 * SIZE(AO), %xmm0 mulpd %xmm0, %xmm1 mulpd 6 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 movapd 4 * SIZE(BO), %xmm1 addpd %xmm0, %xmm9 movapd -10 * SIZE(AO), %xmm0 mulpd %xmm0, %xmm1 mulpd 6 * SIZE(BO), %xmm0 addpd %xmm1, %xmm12 movapd 32 * SIZE(BO), %xmm1 addpd %xmm0, %xmm13 movapd 16 * SIZE(AO), %xmm0 PREFETCH (PREFETCHSIZE + 8) * SIZE(AO) mulpd %xmm2, %xmm3 mulpd 10 * SIZE(BO), %xmm2 addpd %xmm3, %xmm8 movapd 8 * SIZE(BO), %xmm3 addpd %xmm2, %xmm9 movapd -6 * SIZE(AO), %xmm2 mulpd %xmm2, %xmm3 mulpd 10 * SIZE(BO), %xmm2 addpd %xmm3, %xmm12 movapd 12 * SIZE(BO), %xmm3 addpd %xmm2, %xmm13 movapd -4 * SIZE(AO), %xmm2 mulpd %xmm2, %xmm3 mulpd 14 * SIZE(BO), %xmm2 addpd %xmm3, %xmm8 movapd 12 * SIZE(BO), %xmm3 addpd %xmm2, %xmm9 movapd -2 * SIZE(AO), %xmm2 mulpd %xmm2, %xmm3 mulpd 14 * SIZE(BO), %xmm2 addpd %xmm3, %xmm12 movapd 40 * SIZE(BO), %xmm3 addpd %xmm2, %xmm13 movapd 24 * SIZE(AO), %xmm2 PREFETCH (PREFETCHSIZE + 16) * SIZE(AO) mulpd %xmm4, %xmm5 mulpd 18 * SIZE(BO), %xmm4 addpd %xmm5, %xmm8 movapd 16 * SIZE(BO), %xmm5 addpd %xmm4, %xmm9 movapd 2 * SIZE(AO), %xmm4 mulpd %xmm4, %xmm5 mulpd 18 * SIZE(BO), %xmm4 addpd %xmm5, %xmm12 movapd 20 * SIZE(BO), %xmm5 addpd %xmm4, %xmm13 movapd 4 * SIZE(AO), %xmm4 mulpd %xmm4, %xmm5 mulpd 22 * SIZE(BO), %xmm4 addpd %xmm5, %xmm8 movapd 20 * SIZE(BO), %xmm5 addpd %xmm4, %xmm9 movapd 6 * SIZE(AO), %xmm4 mulpd %xmm4, %xmm5 mulpd 22 * SIZE(BO), %xmm4 addpd %xmm5, %xmm12 movapd 48 * SIZE(BO), %xmm5 addpd %xmm4, %xmm13 movapd 32 * SIZE(AO), %xmm4 PREFETCH (PREFETCHSIZE + 24) * SIZE(AO) mulpd %xmm6, %xmm7 mulpd 26 * SIZE(BO), %xmm6 addpd %xmm7, %xmm8 movapd 24 * SIZE(BO), %xmm7 addpd %xmm6, %xmm9 movapd 10 * SIZE(AO), %xmm6 mulpd %xmm6, %xmm7 mulpd 26 * SIZE(BO), %xmm6 addpd %xmm7, %xmm12 movapd 28 * SIZE(BO), %xmm7 addpd %xmm6, %xmm13 movapd 12 * SIZE(AO), %xmm6 mulpd %xmm6, %xmm7 mulpd 30 * SIZE(BO), %xmm6 addpd %xmm7, %xmm8 movapd 28 * SIZE(BO), %xmm7 addpd %xmm6, %xmm9 movapd 14 * SIZE(AO), %xmm6 mulpd %xmm6, %xmm7 mulpd 30 * SIZE(BO), %xmm6 addpd %xmm7, %xmm12 movapd 56 * SIZE(BO), %xmm7 addpd %xmm6, %xmm13 movapd 40 * SIZE(AO), %xmm6 addq $32 * SIZE, AO addq $32 * SIZE, BO decq %rax jne .L52 ALIGN_3.L55:#ifndef TRMMKERNEL movq K, %rax#else movq KKK, %rax#endif movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH je .L59 ALIGN_3.L56: movapd 0 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 addpd %xmm1, %xmm8 mulpd 2 * SIZE(BO), %xmm0 addpd %xmm0, %xmm9 movapd -14 * SIZE(AO), %xmm0 movapd 0 * SIZE(BO), %xmm1 mulpd %xmm0, %xmm1 addpd %xmm1, %xmm12 mulpd 2 * SIZE(BO), %xmm0 addpd %xmm0, %xmm13 movapd -12 * SIZE(AO), %xmm0 addq $4 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax jg .L56 ALIGN_3.L59: movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 movhpd 3 * SIZE(CO1), %xmm1 movsd 4 * SIZE(CO1), %xmm2 movhpd 5 * SIZE(CO1), %xmm2 movsd 6 * SIZE(CO1), %xmm3 movhpd 7 * SIZE(CO1), %xmm3 movddup %xmm8, %xmm4 unpckhpd %xmm8, %xmm8 movddup %xmm12, %xmm5 unpckhpd %xmm12, %xmm12 mulpd %xmm7, %xmm4 mulpd %xmm7, %xmm8 mulpd %xmm7, %xmm5 mulpd %xmm7, %xmm12 addpd %xmm4, %xmm0 addpd %xmm8, %xmm1 addpd %xmm5, %xmm2 addpd %xmm12, %xmm3 movsd %xmm0, 0 * SIZE(CO1) movhpd %xmm0, 1 * SIZE(CO1) movsd %xmm1, 2 * SIZE(CO1) movhpd %xmm1, 3 * SIZE(CO1) movsd %xmm2, 4 * SIZE(CO1) movhpd %xmm2, 5 * SIZE(CO1) movsd %xmm3, 6 * SIZE(CO1) movhpd %xmm3, 7 * SIZE(CO1) movsd 0 * SIZE(CO2), %xmm0 movhpd 1 * SIZE(CO2), %xmm0 movsd 2 * SIZE(CO2), %xmm1 movhpd 3 * SIZE(CO2), %xmm1 movsd 4 * SIZE(CO2), %xmm2 movhpd 5 * SIZE(CO2), %xmm2 movsd 6 * SIZE(CO2), %xmm3 movhpd 7 * SIZE(CO2), %xmm3 movddup %xmm9, %xmm4 unpckhpd %xmm9, %xmm9 movddup %xmm13, %xmm5 unpckhpd %xmm13, %xmm13 mulpd %xmm7, %xmm4 mulpd %xmm7, %xmm9 mulpd %xmm7, %xmm5 mulpd %xmm7, %xmm13 addpd %xmm4, %xmm0 addpd %xmm9, %xmm1 addpd %xmm5, %xmm2 addpd %xmm13, %xmm3 movsd %xmm0, 0 * SIZE(CO2) movhpd %xmm0, 1 * SIZE(CO2) movsd %xmm1, 2 * SIZE(CO2) movhpd %xmm1, 3 * SIZE(CO2) movsd %xmm2, 4 * SIZE(CO2) movhpd %xmm2, 5 * SIZE(CO2) movsd %xmm3, 6 * SIZE(CO2) movhpd %xmm3, 7 * SIZE(CO2) addq $8 * SIZE, CO1 # coffset += 4 addq $8 * SIZE, CO2 # coffset += 4 decq I # i -- jg .L51 ALIGN_3 .L60: testq $2, M je .L70 ALIGN_3.L61:#if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER, BO#else leaq BUFFER, BO movq KK, %rax leaq (, %rax, SIZE), %rax leaq (AO, %rax, 2), AO leaq (BO, %rax, 4), BO#endif movapd -16 * SIZE(AO), %xmm0 pxor %xmm8, %xmm8 movapd 0 * SIZE(BO), %xmm1 pxor %xmm9, %xmm9 movapd -8 * SIZE(AO), %xmm2 pxor %xmm10, %xmm10 movapd 8 * SIZE(BO), %xmm3 pxor %xmm11, %xmm11 movapd 16 * SIZE(BO), %xmm5 movapd 24 * SIZE(BO), %xmm7#ifndef TRMMKERNEL movq K, %rax#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) movq K, %rax subq KK, %rax movq %rax, KKK #else movq KK, %rax#ifdef LEFT addq $2, %rax#else addq $2, %rax#endif movq %rax, KKK#endif sarq $3, %rax je .L65 ALIGN_3.L62: mulpd %xmm0, %xmm1 PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) mulpd 2 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 movapd 4 * SIZE(BO), %xmm1 addpd %xmm0, %xmm9 movapd -14 * SIZE(AO), %xmm0 mulpd %xmm0, %xmm1 mulpd 6 * SIZE(BO), %xmm0 addpd %xmm1, %xmm10 movapd 32 * SIZE(BO), %xmm1 addpd %xmm0, %xmm11 movapd -12 * SIZE(AO), %xmm0 mulpd %xmm0, %xmm3 mulpd 10 * SIZE(BO), %xmm0 addpd %xmm3, %xmm8 movapd 12 * SIZE(BO), %xmm3 addpd %xmm0, %xmm9 movapd -10 * SIZE(AO), %xmm0 mulpd %xmm0, %xmm3 mulpd 14 * SIZE(BO), %xmm0 addpd %xmm3, %xmm10 movapd 40 * SIZE(BO), %xmm3 addpd %xmm0, %xmm11 movapd 0 * SIZE(AO), %xmm0 PREFETCH (PREFETCHSIZE + 8) * SIZE(AO) mulpd %xmm2, %xmm5 mulpd 18 * SIZE(BO), %xmm2 addpd %xmm5, %xmm8 movapd 20 * SIZE(BO), %xmm5 addpd %xmm2, %xmm9 movapd -6 * SIZE(AO), %xmm2 mulpd %xmm2, %xmm5 mulpd 22 * SIZE(BO), %xmm2 addpd %xmm5, %xmm10 movapd 48 * SIZE(BO), %xmm5 addpd %xmm2, %xmm11 movapd -4 * SIZE(AO), %xmm2 mulpd %xmm2, %xmm7 mulpd 26 * SIZE(BO), %xmm2 addpd %xmm7, %xmm8 movapd 28 * SIZE(BO), %xmm7 addpd %xmm2, %xmm9 movapd -2 * SIZE(AO), %xmm2 mulpd %xmm2, %xmm7 mulpd 30 * SIZE(BO), %xmm2 addpd %xmm7, %xmm10 movapd 56 * SIZE(BO), %xmm7 addpd %xmm2, %xmm11 movapd 8 * SIZE(AO), %xmm2 addq $16 * SIZE, AO addq $32 * SIZE, BO decq %rax jne .L62 ALIGN_3.L65:#ifndef TRMMKERNEL movq K, %rax#else movq KKK, %rax#endif movapd ALPHA, %xmm7 andq $7, %rax # if (k & 1) BRANCH je .L69 ALIGN_3.L66: mulpd %xmm0, %xmm1 mulpd 2 * SIZE(BO), %xmm0 addpd %xmm1, %xmm8 movapd 4 * SIZE(BO), %xmm1 addpd %xmm0, %xmm9 movapd -14 * SIZE(AO), %xmm0 addq $2 * SIZE, AO # aoffset += 4 addq $4 * SIZE, BO # boffset1 += 8 decq %rax jg .L66 ALIGN_3.L69: addpd %xmm10, %xmm8 addpd %xmm11, %xmm9 movsd 0 * SIZE(CO1), %xmm0 movhpd 1 * SIZE(CO1), %xmm0 movsd 2 * SIZE(CO1), %xmm1 movhpd 3 * SIZE(CO1), %xmm1 movddup %xmm8, %xmm4 unpckhpd %xmm8, %xmm8 mulpd %xmm7, %xmm4 mulpd %xmm7, %xmm8 addpd %xmm4, %xmm0 addpd %xmm8, %xmm1 movsd %xmm0, 0 * SIZE(CO1) movhpd %xmm0, 1 * SIZE(CO1) movsd %xmm1, 2 * SIZE(CO1) movhpd %xmm1, 3 * SIZE(CO1) movsd 0 * SIZE(CO2), %xmm0 movhpd 1 * SIZE(CO2), %xmm0 movsd 2 * SIZE(CO2), %xmm1 movhpd 3 * SIZE(CO2), %xmm1 movddup %xmm9, %xmm4 unpckhpd %xmm9, %xmm9 mulpd %xmm7, %xmm4 mulpd %xmm7, %xmm9 addpd %xmm4, %xmm0 addpd %xmm9, %xmm1 movsd %xmm0, 0 * SIZE(CO2) movhpd %xmm0, 1 * SIZE(CO2) movsd %xmm1, 2 * SIZE(CO2) movhpd %xmm1, 3 * SIZE(CO2) addq $4 * SIZE, CO1 # coffset += 4 addq $4 * SIZE, CO2 # coffset += 4 ALIGN_3 .L70: testq $1, M je .L79 ALIGN_3.L71:#if !defined(TRMMKERNEL) || \ (defined(TRMMKERNEL) && defined(LEFT) && defined(TRANSA)) || \ (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA)) leaq BUFFER, BO#else leaq BUFFER, BO movq KK, %rax leaq (, %rax, SIZE), %rax leaq (AO, %rax, 1), AO leaq (BO, %rax, 4), BO#endif movsd -16 * SIZE(AO), %xmm0 pxor %xmm8, %xmm8 movsd 0 * SIZE(BO), %xmm1 pxor %xmm9, %xmm9 movsd -12 * SIZE(AO), %xmm2 pxor %xmm10, %xmm10
⌨️ 快捷键说明
复制代码Ctrl + C
搜索代码Ctrl + F
全屏模式F11
增大字号Ctrl + =
减小字号Ctrl + -
显示快捷键?