skl_dct_sse.asm

mpeg4 video codec mpeg4 video codec

;/********************************************************
; * Some code. Copyright (C) 2003 by Pascal Massimino.   *
; * All Rights Reserved.      (http://skal.planet-d.net) *
; * For Educational/Academic use ONLY. See 'LICENSE.TXT'.*
; ********************************************************/
; [BITS 32]

%include "skl_nasm.h"

;//////////////////////////////////////////////////////////////////////

%macro cfastcall 1 
		global @%1@4
		%define %1 @%1@4
%endmacro

;cfastcall Skl_IDct16_SSE
globl Skl_IDct16_SSE
globl Skl_IDct16_MMX
globl Skl_IDct16_Put_SSE
globl Skl_IDct16_Put_MMX
globl Skl_IDct16_Add_SSE
globl Skl_IDct16_Add_MMX

;//////////////////////////////////////////////////////////////////////
;
;                          -=FDCT=-
;
; Vertical pass is an implementation of the scheme:
;  Loeffler C., Ligtenberg A., and Moschytz C.S.:
;  Practical Fast 1D DCT Algorithm with Eleven Multiplications,
;  Proc. ICASSP 1989, 988-991.
;
; Horizontal pass is a double 4x4 vector/matrix multiplication,
; (see also Intel's Application Note 922:
;  http://developer.intel.com/vtune/cbts/strmsimd/922down.htm
;  Copyright (C) 1999 Intel Corporation)
;  
; Notes:
;  * tan(3pi/16) is greater than 0.5, and would use the
;    sign bit when turned into 16b fixed-point precision. So,
;    we use the trick: x*tan3 = x*(tan3-1)+x
; 
;  * There's only one SSE-specific instruction (pshufw).
;
;  * There's still 1 or 2 ticks to save in fLLM_PASS, but
;    I prefer having a readable code, instead of a tightly
;    scheduled one...
;
;  * Quantization stage (as well as pre-transposition for the
;    idct way back) can be included in the fTab* constants
;    (with induced loss of precision, somehow)
;
;  * Some more details at: http://skal.planet-d.net/coding/dct.html
;
;
;//////////////////////////////////////////////////////////////////////
;
;  == Mean square errors ==
;   0.000 0.001 0.001 0.002 0.000 0.002 0.001 0.000    [0.001]
;   0.035 0.029 0.032 0.032 0.031 0.032 0.034 0.035    [0.032]
;   0.026 0.028 0.027 0.027 0.025 0.028 0.028 0.025    [0.027]
;   0.037 0.032 0.031 0.030 0.028 0.029 0.026 0.031    [0.030]
;   0.000 0.001 0.001 0.002 0.000 0.002 0.001 0.001    [0.001]
;   0.025 0.024 0.022 0.022 0.022 0.022 0.023 0.023    [0.023]
;   0.026 0.028 0.025 0.028 0.030 0.025 0.026 0.027    [0.027]
;   0.021 0.020 0.020 0.022 0.020 0.022 0.017 0.019    [0.020]
;  
;  == Abs Mean errors ==
;   0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000    [0.000]
;   0.020 0.001 0.003 0.003 0.000 0.004 0.002 0.003    [0.002]
;   0.000 0.001 0.001 0.001 0.001 0.004 0.000 0.000    [0.000]
;   0.027 0.001 0.000 0.002 0.002 0.002 0.001 0.000    [0.003]
;   0.000 0.000 0.000 0.000 0.000 0.001 0.000 0.001    [-0.000]
;   0.001 0.003 0.001 0.001 0.002 0.001 0.000 0.000    [-0.000]
;   0.000 0.002 0.002 0.001 0.001 0.002 0.001 0.000    [-0.000]
;   0.000 0.002 0.001 0.002 0.001 0.002 0.001 0.001    [-0.000]
;
;  =========================
;  Peak error:   1.0000
;  Peak MSE:     0.0365
;  Overall MSE:  0.0201
;  Peak ME:      0.0265
;  Overall ME:   0.0006
;
;//////////////////////////////////////////////////////////////////////
;
;                          -=IDCT=-
;
; A little slower than fdct, because the final stages (butterflies and
; descaling) require some unpairable shifting and packing, all on
; the same CPU unit.
;
;   THIS IDCT IS NOT IEEE-COMPLIANT: IT WILL FAIL THE [-300,300]
;   INPUT RANGE TEST (because of overflow). But the [-256,255] one
;   is OK, and I'm fine with it (for now;)
;
;   If you want ultra-compliant iDCT, just %define IEEE_COMPLIANT.
;   The code will then pass the [-384,383] input test.
;
;  == Mean square errors ==
;   0.007 0.006 0.005 0.007 0.006 0.007 0.005 0.007    [0.006]
;   0.006 0.008 0.007 0.007 0.007 0.008 0.008 0.008    [0.007]
;   0.008 0.008 0.008 0.008 0.007 0.009 0.010 0.007    [0.008]
;   0.007 0.007 0.006 0.007 0.008 0.007 0.006 0.008    [0.007]
;   0.007 0.006 0.006 0.006 0.006 0.005 0.006 0.006    [0.006]
;   0.008 0.007 0.006 0.008 0.007 0.008 0.009 0.009    [0.008]
;   0.008 0.006 0.010 0.008 0.008 0.008 0.007 0.007    [0.008]
;   0.007 0.006 0.006 0.007 0.007 0.006 0.006 0.007    [0.006]
;  
;  == Abs Mean errors ==
;   0.001 0.000 0.000 0.001 0.001 0.000 0.000 0.000    [0.000]
;   0.000 0.002 0.002 0.000 0.001 0.001 0.000 0.002    [0.000]
;   0.001 0.002 0.001 0.001 0.001 0.001 0.000 0.001    [-0.001]
;   0.000 0.002 0.000 0.000 0.001 0.000 0.000 0.001    [-0.000]
;   0.000 0.001 0.001 0.001 0.000 0.001 0.000 0.001    [0.000]
;   0.000 0.001 0.001 0.001 0.001 0.000 0.001 0.000    [0.000]
;   0.001 0.001 0.002 0.001 0.001 0.002 0.001 0.001    [0.001]
;   0.000 0.000 0.001 0.000 0.000 0.000 0.000 0.000    [0.000]
;
;  =========================
;
;  Peak error:   1.0000
;  Peak MSE:     0.0092
;  Overall MSE:  0.0071
;  Peak ME:      0.0022
;  Overall ME:  -0.0002
;
;//////////////////////////////////////////////////////////////////////

DATA

align 16
tan1:    dw  0x32ec,0x32ec,0x32ec,0x32ec    ; tan( pi/16)
tan2:    dw  0x6a0a,0x6a0a,0x6a0a,0x6a0a    ; tan(2pi/16)  (=sqrt(2)-1)
tan3:    dw  0xab0e,0xab0e,0xab0e,0xab0e    ; tan(3pi/16)-1
sqrt2:   dw  0x5a82,0x5a82,0x5a82,0x5a82    ; 0.5/sqrt(2)

;//////////////////////////////////////////////////////////////////////

align 16
iTab1:
  dw 0x4000, 0x539f, 0x4000, 0x22a3,
  dw 0x4000, 0x22a3, 0xc000, 0xac61,
  dw 0x4000, 0xdd5d, 0x4000, 0xac61,
  dw 0xc000, 0x539f, 0x4000, 0xdd5d,
  dw 0x58c5, 0x4b42, 0x4b42, 0xee58,
  dw 0x3249, 0x11a8, 0xa73b, 0xcdb7,
  dw 0x3249, 0xa73b, 0x11a8, 0xcdb7,
  dw 0x11a8, 0x4b42, 0x4b42, 0xa73b

iTab2:
  dw 0x58c5, 0x73fc, 0x58c5, 0x300b,
  dw 0x58c5, 0x300b, 0xa73b, 0x8c04,
  dw 0x58c5, 0xcff5, 0x58c5, 0x8c04,
  dw 0xa73b, 0x73fc, 0x58c5, 0xcff5,
  dw 0x7b21, 0x6862, 0x6862, 0xe782,
  dw 0x45bf, 0x187e, 0x84df, 0xba41,
  dw 0x45bf, 0x84df, 0x187e, 0xba41,
  dw 0x187e, 0x6862, 0x6862, 0x84df

iTab3:
  dw 0x539f, 0x6d41, 0x539f, 0x2d41,
  dw 0x539f, 0x2d41, 0xac61, 0x92bf,
  dw 0x539f, 0xd2bf, 0x539f, 0x92bf,
  dw 0xac61, 0x6d41, 0x539f, 0xd2bf,
  dw 0x73fc, 0x6254, 0x6254, 0xe8ee,
  dw 0x41b3, 0x1712, 0x8c04, 0xbe4d,
  dw 0x41b3, 0x8c04, 0x1712, 0xbe4d,
  dw 0x1712, 0x6254, 0x6254, 0x8c04

iTab4:
  dw 0x4b42, 0x6254, 0x4b42, 0x28ba,
  dw 0x4b42, 0x28ba, 0xb4be, 0x9dac,
  dw 0x4b42, 0xd746, 0x4b42, 0x9dac,
  dw 0xb4be, 0x6254, 0x4b42, 0xd746,
  dw 0x6862, 0x587e, 0x587e, 0xeb3d,
  dw 0x3b21, 0x14c3, 0x979e, 0xc4df,
  dw 0x3b21, 0x979e, 0x14c3, 0xc4df,
  dw 0x14c3, 0x587e, 0x587e, 0x979e

align 16
iTab1_MMX:
  dw 0x4000, 0x4000, 0x4000, 0xc000
  dw 0x539f, 0x22a3, 0x22a3, 0xac61
  dw 0x4000, 0xc000, 0x4000, 0x4000
  dw 0xdd5d, 0x539f, 0xac61, 0xdd5d
  dw 0x58c5, 0x3249, 0x4b42, 0xa73b
  dw 0x4b42, 0x11a8, 0xee58, 0xcdb7
  dw 0x3249, 0x11a8, 0x11a8, 0x4b42
  dw 0xa73b, 0x4b42, 0xcdb7, 0xa73b

iTab2_MMX:
  dw 0x58c5, 0x58c5, 0x58c5, 0xa73b
  dw 0x73fc, 0x300b, 0x300b, 0x8c04
  dw 0x58c5, 0xa73b, 0x58c5, 0x58c5
  dw 0xcff5, 0x73fc, 0x8c04, 0xcff5
  dw 0x7b21, 0x45bf, 0x6862, 0x84df
  dw 0x6862, 0x187e, 0xe782, 0xba41
  dw 0x45bf, 0x187e, 0x187e, 0x6862
  dw 0x84df, 0x6862, 0xba41, 0x84df

iTab3_MMX:
  dw 0x539f, 0x539f, 0x539f, 0xac61
  dw 0x6d41, 0x2d41, 0x2d41, 0x92bf
  dw 0x539f, 0xac61, 0x539f, 0x539f
  dw 0xd2bf, 0x6d41, 0x92bf, 0xd2bf
  dw 0x73fc, 0x41b3, 0x6254, 0x8c04
  dw 0x6254, 0x1712, 0xe8ee, 0xbe4d
  dw 0x41b3, 0x1712, 0x1712, 0x6254
  dw 0x8c04, 0x6254, 0xbe4d, 0x8c04

iTab4_MMX:
  dw 0x4b42, 0x4b42, 0x4b42, 0xb4be
  dw 0x6254, 0x28ba, 0x28ba, 0x9dac
  dw 0x4b42, 0xb4be, 0x4b42, 0x4b42
  dw 0xd746, 0x6254, 0x9dac, 0xd746
  dw 0x6862, 0x3b21, 0x587e, 0x979e
  dw 0x587e, 0x14c3, 0xeb3d, 0xc4df
  dw 0x3b21, 0x14c3, 0x14c3, 0x587e
  dw 0x979e, 0x587e, 0xc4df, 0x979e

  ; the original rounding trick is by
  ; Michel Lespinasse (hi Walken!) <[e-mail]>

align 16
Idct_Rnd0      dd  65535, 65535
Idct_Rnd1      dd   3612,  3612
Idct_Rnd2      dd   2271,  2271
Idct_Rnd3      dd   1203,  1203
Idct_Rnd4      dd   1023,  1023
Idct_Rnd5      dd    102,   102
Idct_Rnd6      dd    398,   398
Idct_Rnd7      dd    469,   469

Idct_Sparse_Rnd0  times 4 dw  (65535>>11)
Idct_Sparse_Rnd1  times 4 dw  ( 3612>>11)
Idct_Sparse_Rnd2  times 4 dw  ( 2271>>11)
  ; other rounders are zero...

;//////////////////////////////////////////////////////////////////////

align 16
fTab1:
  dw 0x4000, 0x4000, 0x58c5, 0x4b42,
  dw 0x4000, 0x4000, 0x3249, 0x11a8,
  dw 0x539f, 0x22a3, 0x4b42, 0xee58,
  dw 0xdd5d, 0xac61, 0xa73b, 0xcdb7,
  dw 0x4000, 0xc000, 0x3249, 0xa73b,
  dw 0xc000, 0x4000, 0x11a8, 0x4b42,
  dw 0x22a3, 0xac61, 0x11a8, 0xcdb7,
  dw 0x539f, 0xdd5d, 0x4b42, 0xa73b

fTab2:
  dw 0x58c5, 0x58c5, 0x7b21, 0x6862,
  dw 0x58c5, 0x58c5, 0x45bf, 0x187e,
  dw 0x73fc, 0x300b, 0x6862, 0xe782,
  dw 0xcff5, 0x8c04, 0x84df, 0xba41,
  dw 0x58c5, 0xa73b, 0x45bf, 0x84df,
  dw 0xa73b, 0x58c5, 0x187e, 0x6862,
  dw 0x300b, 0x8c04, 0x187e, 0xba41,
  dw 0x73fc, 0xcff5, 0x6862, 0x84df

fTab3:
  dw 0x539f, 0x539f, 0x73fc, 0x6254,
  dw 0x539f, 0x539f, 0x41b3, 0x1712,
  dw 0x6d41, 0x2d41, 0x6254, 0xe8ee,
  dw 0xd2bf, 0x92bf, 0x8c04, 0xbe4d,
  dw 0x539f, 0xac61, 0x41b3, 0x8c04,
  dw 0xac61, 0x539f, 0x1712, 0x6254,
  dw 0x2d41, 0x92bf, 0x1712, 0xbe4d,
  dw 0x6d41, 0xd2bf, 0x6254, 0x8c04

fTab4:
  dw 0x4b42, 0x4b42, 0x6862, 0x587e,
  dw 0x4b42, 0x4b42, 0x3b21, 0x14c3,
  dw 0x6254, 0x28ba, 0x587e, 0xeb3d,
  dw 0xd746, 0x9dac, 0x979e, 0xc4df,
  dw 0x4b42, 0xb4be, 0x3b21, 0x979e,
  dw 0xb4be, 0x4b42, 0x14c3, 0x587e,
  dw 0x28ba, 0x9dac, 0x14c3, 0xc4df,
  dw 0x6254, 0xd746, 0x587e, 0x979e

align 16
Fdct_Rnd0: dw  6,8,8,8
Fdct_Rnd1: dw  8,8,8,8
Fdct_Rnd2: dw 10,8,8,8
MMX_One:   dw  1,1,1,1

;//////////////////////////////////////////////////////////////////////

TEXT

;//////////////////////////////////////////////////////////////////////
; iMTX_MULT (~24c)
;//////////////////////////////////////////////////////////////////////

%macro iMTX_MULT 4   ; %1=src, %2 = Table to use, %3=rounder, %4=Shift
  movq mm0, [ecx+%1*16+0]     ; mm0 = [0123]
  movq mm1, [ecx+%1*16+8]     ; mm1 = [4567]

  movq    mm3, [%2+0]   ;  [   M00    M01      M04    M05]
  pshufw  mm2, mm0, 11011101b ;  [1313]
  movq    mm4, [%2+8]   ;  [   M02    M03      M06    M07]
  pshufw  mm0, mm0, 10001000b ;  [0202]
  movq    mm6, [%2+32]  ;  [   M16    M17      M20    M21]
  pshufw  mm5, mm1, 11011101b ;  [5757]
  movq    mm7, [%2+40]  ;  [   M18    M19      M22    M23]
  pshufw  mm1, mm1, 10001000b ;  [4646]
  pmaddwd mm3, mm0      ;  [i0.M00+i2.M01 | i0.M04+i2.M05]
  pmaddwd mm6, mm2      ;  [i1.M16+i3.M17 | i1.M20+i3.M21]
  pmaddwd mm4, mm1      ;  [i4.M02+i6.M03 | i4.M06+i6.M07]
  pmaddwd mm7, mm5      ;  [i5.M18+i7.M19 | i5.M22+i7.M23]
  pmaddwd mm2, [%2+48]  ;  [i1.M24+i3.M25 | i1.M28+i3.M29]
  pmaddwd mm5, [%2+56]  ;  [i5.M26+i7.M27 | i5.M30+i7.M31]
  pmaddwd mm0, [%2+16]  ;  [i0.M08+i2.M09 | i0.M12+i2.M13]
  pmaddwd mm1, [%2+24]  ;  [i4.M10+i6.M11 | i4.M14+i6.M15]

  paddd   mm3, [%3]     ;  Round
  paddd   mm6, mm7      ;  => b0 | b1
  paddd   mm0, [%3]     ;  Round
  paddd   mm2, mm5      ;  => b2 | b3
  paddd   mm3, mm4      ;  => a0 | a1
  paddd   mm0, mm1      ;  => a2 | a3

  movq    mm4, mm3      ;     a0 | a1
  movq    mm7, mm0      ;     a2 | a3
  paddd   mm3, mm6      ;  a0+b0 | a1+b1
  psubd   mm4, mm6      ;  a0-b0 | a1-b1
  psubd   mm7, mm2      ;  a2-b2 | a3-b3
  paddd   mm0, mm2      ;  a2+b2 | a3+b3

  psrad   mm3, %4       ;  => out0 | out1
  psrad   mm4, %4       ;  => out7 | out6
  psrad   mm0, %4       ;  => out2 | out3
  psrad   mm7, %4       ;  => out5 | out4

  packssdw  mm3, mm0            ;  [0123]
  packssdw  mm7, mm4            ;  [5476]

  movq    [ecx+%1*16+0], mm3
  pshufw  mm7, mm7, 10110001b   ;  [4567]

  movq    [ecx+%1*16+8], mm7
%endmacro

%macro iMTX_MULT_03 4   ; %1=src, %2 = Table to use, %3=rounder, %4=Shift
    ; this version assume [4567] coeffs are zero...
  movq mm0, [ecx+%1*16+0]     ; mm0 = [0123]

  movq    mm3, [%2+0]   ;  [   M00    M01      M04    M05]
  pshufw  mm2, mm0, 11011101b ;  [1313]
  movq    mm6, [%2+32]  ;  [   M16    M17      M20    M21]
  pshufw  mm0, mm0, 10001000b ;  [0202]

  pmaddwd mm6, mm2      ;  [i1.M16+i3.M17 | i1.M20+i3.M21]
  pmaddwd mm3, mm0      ;  [i0.M00+i2.M01 | i0.M04+i2.M05]
  pmaddwd mm2, [%2+48]  ;  [i1.M24+i3.M25 | i1.M28+i3.M29]
  pmaddwd mm0, [%2+16]  ;  [i0.M08+i2.M09 | i0.M12+i2.M13]

    ; mm2=b2|b3  mm6  = b0|b1
  paddd   mm3, [%3]     ;  Round
  paddd   mm0, [%3]     ;  Round

  movq    mm4, mm3      ;     a0 | a1
  movq    mm7, mm0      ;     a2 | a3
  paddd   mm3, mm6      ;  a0+b0 | a1+b1
  psubd   mm4, mm6      ;  a0-b0 | a1-b1
  psubd   mm7, mm2      ;  a2-b2 | a3-b3
  paddd   mm0, mm2      ;  a2+b2 | a3+b3

  psrad   mm3, %4       ;  => out0 | out1
  psrad   mm4, %4       ;  => out7 | out6
  psrad   mm0, %4       ;  => out2 | out3
  psrad   mm7, %4       ;  => out5 | out4

  packssdw  mm3, mm0            ;  [0123]
  packssdw  mm7, mm4            ;  [5476]

  movq    [ecx+%1*16+0], mm3
  pshufw  mm7, mm7, 10110001b   ;  [4567]