nb_kernel314_x86_64_sse.s

来自「最著名最快的分子模拟软件」· S 代码 · 共 2,211 行 · 第 1/5 页

    movss  44(%rsi,%rcx,4),%xmm3    ## jzMc  -  -  -
    movss  44(%rsi,%rbx,4),%xmm5    ## jzMb  -  -  -
    movss  44(%rsi,%rdx,4),%xmm6    ## jzMd  -  -  -
    movlhps %xmm5,%xmm2 ## jzMa  -  jzMb  -
    movlhps %xmm6,%xmm3 ## jzMc  -  jzMd -

    movaps %xmm0,%xmm4
    unpcklps %xmm1,%xmm0 ## jxMa jxMc jyMa jyMc        
    unpckhps %xmm1,%xmm4 ## jxMb jxMd jyMb jyMd
    movaps %xmm0,%xmm1
    unpcklps %xmm4,%xmm0 ## x
    unpckhps %xmm4,%xmm1 ## y

    shufps  $136,%xmm3,%xmm2  ## 10001000 => jzMa jzMb jzMc jzMd

    ## xmm0 = Mx
    ## xmm1 = My
    ## xmm2 = Mz

    movaps %xmm0,%xmm3
    movaps %xmm1,%xmm4
    movaps %xmm2,%xmm5
    movaps %xmm0,%xmm6
    movaps %xmm1,%xmm7
    movaps %xmm2,%xmm8

    subps nb314_ixH1(%rsp),%xmm0
    subps nb314_iyH1(%rsp),%xmm1
    subps nb314_izH1(%rsp),%xmm2
    subps nb314_ixH2(%rsp),%xmm3
    subps nb314_iyH2(%rsp),%xmm4
    subps nb314_izH2(%rsp),%xmm5
    subps nb314_ixM(%rsp),%xmm6
    subps nb314_iyM(%rsp),%xmm7
    subps nb314_izM(%rsp),%xmm8

        movaps %xmm0,nb314_dxH1M(%rsp)
        movaps %xmm1,nb314_dyH1M(%rsp)
        movaps %xmm2,nb314_dzH1M(%rsp)
        mulps  %xmm0,%xmm0
        mulps  %xmm1,%xmm1
        mulps  %xmm2,%xmm2
        movaps %xmm3,nb314_dxH2M(%rsp)
        movaps %xmm4,nb314_dyH2M(%rsp)
        movaps %xmm5,nb314_dzH2M(%rsp)
        mulps  %xmm3,%xmm3
        mulps  %xmm4,%xmm4
        mulps  %xmm5,%xmm5
        movaps %xmm6,nb314_dxMM(%rsp)
        movaps %xmm7,nb314_dyMM(%rsp)
        movaps %xmm8,nb314_dzMM(%rsp)
        mulps  %xmm6,%xmm6
        mulps  %xmm7,%xmm7
        mulps  %xmm8,%xmm8
        addps  %xmm1,%xmm0
        addps  %xmm2,%xmm0
        addps  %xmm4,%xmm3
        addps  %xmm5,%xmm3
    addps  %xmm7,%xmm6
    addps  %xmm8,%xmm6

        ## start doing invsqrt for jM atoms
        rsqrtps %xmm0,%xmm1
        rsqrtps %xmm3,%xmm4
    rsqrtps %xmm6,%xmm7

        movaps  %xmm1,%xmm2
        movaps  %xmm4,%xmm5
    movaps  %xmm7,%xmm8

        mulps   %xmm1,%xmm1 ## lu*lu
        mulps   %xmm4,%xmm4 ## lu*lu
    mulps   %xmm7,%xmm7 ## lu*lu

        movaps  nb314_three(%rsp),%xmm9
        movaps  %xmm9,%xmm10
    movaps  %xmm9,%xmm11

        mulps   %xmm0,%xmm1 ## rsq*lu*lu
        mulps   %xmm3,%xmm4 ## rsq*lu*lu 
    mulps   %xmm6,%xmm7 ## rsq*lu*lu

        subps   %xmm1,%xmm9
        subps   %xmm4,%xmm10
    subps   %xmm7,%xmm11 ## 3-rsq*lu*lu

        mulps   %xmm2,%xmm9
        mulps   %xmm5,%xmm10
    mulps   %xmm8,%xmm11 ## lu*(3-rsq*lu*lu)

        movaps  nb314_half(%rsp),%xmm4
        mulps   %xmm4,%xmm9 ## rinvH1M
        mulps   %xmm4,%xmm10 ## rinvH2M
    mulps   %xmm4,%xmm11 ## rinvMM

        movaps  %xmm9,nb314_rinvH1M(%rsp)
        movaps  %xmm10,nb314_rinvH2M(%rsp)
        movaps  %xmm11,nb314_rinvMM(%rsp)

        ## M interactions 
    ## rsq in xmm0,xmm3,xmm6  
    ## rinv in xmm9, xmm10, xmm11

    movaps nb314_tsc(%rsp),%xmm1
    mulps  %xmm9,%xmm0 ## r
    mulps  %xmm10,%xmm3
    mulps  %xmm11,%xmm6
    mulps  %xmm1,%xmm0 ## rtab
    mulps  %xmm1,%xmm3
    mulps  %xmm1,%xmm6

    ## truncate and convert to integers
    cvttps2dq %xmm0,%xmm1
    cvttps2dq %xmm3,%xmm4
    cvttps2dq %xmm6,%xmm7

    ## convert back to float
    cvtdq2ps  %xmm1,%xmm2
    cvtdq2ps  %xmm4,%xmm5
    cvtdq2ps  %xmm7,%xmm8

    ## multiply by 4
    pslld   $2,%xmm1
    pslld   $2,%xmm4
    pslld   $2,%xmm7

    ## move to integer registers
    movhlps %xmm1,%xmm13
    movhlps %xmm4,%xmm14
    movhlps %xmm7,%xmm15
    movd    %xmm1,%eax
    movd    %xmm4,%r8d
    movd    %xmm7,%r12d
    movd    %xmm13,%ecx
    movd    %xmm14,%r10d
    movd    %xmm15,%r14d
    pshufd $1,%xmm1,%xmm1
    pshufd $1,%xmm4,%xmm4
    pshufd $1,%xmm7,%xmm7
    pshufd $1,%xmm13,%xmm13
    pshufd $1,%xmm14,%xmm14
    pshufd $1,%xmm15,%xmm15
    movd    %xmm1,%ebx
    movd    %xmm4,%r9d
    movd    %xmm7,%r13d
    movd    %xmm13,%edx
    movd    %xmm14,%r11d
    movd    %xmm15,%r15d

    movq nb314_VFtab(%rbp),%rsi

    ## calculate eps
    subps     %xmm2,%xmm0
    subps     %xmm5,%xmm3
    subps     %xmm8,%xmm6

    movaps    %xmm0,nb314_epsH1(%rsp)
    movaps    %xmm3,nb314_epsH2(%rsp)
    movaps    %xmm6,nb314_epsM(%rsp)

    ## Load LOTS of table data
        movlps (%rsi,%rax,4),%xmm1
        movlps (%rsi,%r8,4),%xmm5
        movlps (%rsi,%r12,4),%xmm9

        movlps (%rsi,%rcx,4),%xmm3
        movlps (%rsi,%r10,4),%xmm7
        movlps (%rsi,%r14,4),%xmm11

        movhps (%rsi,%rbx,4),%xmm1
        movhps (%rsi,%r9,4),%xmm5
        movhps (%rsi,%r13,4),%xmm9

        movhps (%rsi,%rdx,4),%xmm3
        movhps (%rsi,%r11,4),%xmm7
        movhps (%rsi,%r15,4),%xmm11

    movaps %xmm1,%xmm0
    movaps %xmm5,%xmm4
    movaps %xmm9,%xmm8
        shufps $136,%xmm3,%xmm0 ## 10001000
        shufps $136,%xmm7,%xmm4 ## 10001000
        shufps $136,%xmm11,%xmm8 ## 10001000
        shufps $221,%xmm3,%xmm1 ## 11011101
        shufps $221,%xmm7,%xmm5 ## 11011101
        shufps $221,%xmm11,%xmm9 ## 11011101

        movlps 8(%rsi,%rax,4),%xmm3
        movlps 8(%rsi,%r8,4),%xmm7
        movlps 8(%rsi,%r12,4),%xmm11

        movlps 8(%rsi,%rcx,4),%xmm12
        movlps 8(%rsi,%r10,4),%xmm13
        movlps 8(%rsi,%r14,4),%xmm14

        movhps 8(%rsi,%rbx,4),%xmm3
        movhps 8(%rsi,%r9,4),%xmm7
        movhps 8(%rsi,%r13,4),%xmm11

        movhps 8(%rsi,%rdx,4),%xmm12
        movhps 8(%rsi,%r11,4),%xmm13
        movhps 8(%rsi,%r15,4),%xmm14

    movaps %xmm3,%xmm2
    movaps %xmm7,%xmm6
    movaps %xmm11,%xmm10

        shufps $136,%xmm12,%xmm2 ## 10001000
        shufps $136,%xmm13,%xmm6 ## 10001000
        shufps $136,%xmm14,%xmm10 ## 10001000
        shufps $221,%xmm12,%xmm3 ## 11011101
        shufps $221,%xmm13,%xmm7 ## 11011101
        shufps $221,%xmm14,%xmm11 ## 11011101
    ## table data ready in xmm0-xmm3 , xmm4-xmm7 , and xmm8-xmm11

    movaps nb314_epsH1(%rsp),%xmm12
    movaps nb314_epsH2(%rsp),%xmm13
    movaps nb314_epsM(%rsp),%xmm14

    mulps  %xmm12,%xmm3  ## Heps
    mulps  %xmm13,%xmm7
    mulps  %xmm14,%xmm11
    mulps  %xmm12,%xmm2  ## Geps
    mulps  %xmm13,%xmm6
    mulps  %xmm14,%xmm10
    mulps  %xmm12,%xmm3  ## Heps2
    mulps  %xmm13,%xmm7
    mulps  %xmm14,%xmm11

    addps  %xmm2,%xmm1  ## F+Geps
    addps  %xmm6,%xmm5
    addps  %xmm10,%xmm9
    addps  %xmm3,%xmm1  ## F+Geps+Heps2 = Fp
    addps  %xmm7,%xmm5
    addps  %xmm11,%xmm9
    addps  %xmm3,%xmm3   ## 2*Heps2
    addps  %xmm7,%xmm7
    addps  %xmm11,%xmm11
    addps  %xmm2,%xmm3   ## 2*Heps2+Geps
    addps  %xmm6,%xmm7
    addps  %xmm10,%xmm11
    addps  %xmm1,%xmm3  ## FF = Fp + 2*Heps2 + Geps
    addps  %xmm5,%xmm7
    addps  %xmm9,%xmm11
    mulps  %xmm12,%xmm1  ## eps*Fp
    mulps  %xmm13,%xmm5
    mulps  %xmm14,%xmm9
    movaps nb314_qqMH(%rsp),%xmm12
    movaps nb314_qqMM(%rsp),%xmm13
    addps  %xmm0,%xmm1    ## VV
    addps  %xmm4,%xmm5
    addps  %xmm8,%xmm9
    mulps  %xmm12,%xmm1  ## VV*qq = vcoul
    mulps  %xmm12,%xmm5
    mulps  %xmm13,%xmm9
    mulps  %xmm12,%xmm3   ## FF*qq = fij
    mulps  %xmm12,%xmm7
    mulps  %xmm13,%xmm11

    ## accumulate vctot
    addps  nb314_vctot(%rsp),%xmm1
    addps  %xmm9,%xmm5
    addps  %xmm5,%xmm1
    movaps %xmm1,nb314_vctot(%rsp)

    movaps nb314_tsc(%rsp),%xmm10
    mulps  %xmm10,%xmm3 ## fscal
    mulps  %xmm10,%xmm7
    mulps  %xmm11,%xmm10

    movd %mm0,%eax ## restore j3 from mm0-mm3
    movd %mm1,%ebx
    movd %mm2,%ecx
    movd %mm3,%edx

        ## move j M forces to local temp variables 
    movlps 36(%rdi,%rax,4),%xmm11    ## jxMa jyMa  -   -
    movlps 36(%rdi,%rcx,4),%xmm12    ## jxMc jyMc  -   -
    movhps 36(%rdi,%rbx,4),%xmm11    ## jxMa jyMa jxMb jyMb 
    movhps 36(%rdi,%rdx,4),%xmm12    ## jxMc jyMc jxMd jyMd 

    movss  44(%rdi,%rax,4),%xmm13    ## jzMa  -  -  -
    movss  44(%rdi,%rcx,4),%xmm14    ## jzMc  -  -  -
    movss  44(%rdi,%rbx,4),%xmm1     ## jzMb  -  -  -
    movss  44(%rdi,%rdx,4),%xmm2     ## jzMd  -  -  -
    movlhps %xmm1,%xmm13 ## jzMa  -  jzMb  -
    movlhps %xmm2,%xmm14 ## jzMc  -  jzMd -

    shufps $136,%xmm14,%xmm13 ## 10001000 => jzMa jzMb jzMc jzMd

    ## xmm11: jxMa jyMa jxMb jyMb 
    ## xmm12: jxMc jyMc jxMd jyMd
    ## xmm13: jzMa jzMb jzMc jzMd

    xorps  %xmm0,%xmm0
    xorps  %xmm4,%xmm4
    xorps  %xmm8,%xmm8

    mulps  nb314_rinvH1M(%rsp),%xmm3
    mulps  nb314_rinvH2M(%rsp),%xmm7
    mulps  nb314_rinvMM(%rsp),%xmm10

    subps  %xmm3,%xmm0
    subps  %xmm7,%xmm4
    subps  %xmm10,%xmm8

    movaps %xmm0,%xmm1
    movaps %xmm0,%xmm2
    movaps %xmm4,%xmm3
    movaps %xmm4,%xmm5
    movaps %xmm8,%xmm6
    movaps %xmm8,%xmm7

        mulps nb314_dxH1M(%rsp),%xmm0
        mulps nb314_dyH1M(%rsp),%xmm1
        mulps nb314_dzH1M(%rsp),%xmm2
        mulps nb314_dxH2M(%rsp),%xmm3
        mulps nb314_dyH2M(%rsp),%xmm4
        mulps nb314_dzH2M(%rsp),%xmm5
        mulps nb314_dxMM(%rsp),%xmm6
        mulps nb314_dyMM(%rsp),%xmm7
        mulps nb314_dzMM(%rsp),%xmm8

    movaps %xmm0,%xmm14
    movaps %xmm1,%xmm15
    addps %xmm2,%xmm13
    addps nb314_fixH1(%rsp),%xmm0
    addps nb314_fiyH1(%rsp),%xmm1
    addps nb314_fizH1(%rsp),%xmm2

    addps %xmm3,%xmm14
    addps %xmm4,%xmm15
    addps %xmm5,%xmm13
    addps nb314_fixH2(%rsp),%xmm3
    addps nb314_fiyH2(%rsp),%xmm4
    addps nb314_fizH2(%rsp),%xmm5

    addps %xmm6,%xmm14
    addps %xmm7,%xmm15
    addps %xmm8,%xmm13
    addps nb314_fixM(%rsp),%xmm6
    addps nb314_fiyM(%rsp),%xmm7
    addps nb314_fizM(%rsp),%xmm8

    movaps %xmm0,nb314_fixH1(%rsp)
    movaps %xmm1,nb314_fiyH1(%rsp)
    movaps %xmm2,nb314_fizH1(%rsp)
    movaps %xmm3,nb314_fixH2(%rsp)
    movaps %xmm4,nb314_fiyH2(%rsp)
    movaps %xmm5,nb314_fizH2(%rsp)
    movaps %xmm6,nb314_fixM(%rsp)
    movaps %xmm7,nb314_fiyM(%rsp)
    movaps %xmm8,nb314_fizM(%rsp)

    ## xmm14 = fMx
    ## xmm15 = fMy
    ## xmm13 = fMz
    movaps %xmm14,%xmm0
    unpcklps %xmm15,%xmm14
    unpckhps %xmm15,%xmm0

    addps  %xmm14,%xmm11
    addps  %xmm0,%xmm12

    movhlps  %xmm13,%xmm14 ## fMzc fMzd

    movlps %xmm11,36(%rdi,%rax,4)
    movhps %xmm11,36(%rdi,%rbx,4)
    movlps %xmm12,36(%rdi,%rcx,4)
    movhps %xmm12,36(%rdi,%rdx,4)
    movss  %xmm13,44(%rdi,%rax,4)
    movss  %xmm14,44(%rdi,%rcx,4)
    shufps $1,%xmm13,%xmm13
    shufps $1,%xmm14,%xmm14
    movss  %xmm13,44(%rdi,%rbx,4)
    movss  %xmm14,44(%rdi,%rdx,4)

        ## should we do one more iteration? 
        subl $4,nb314_innerk(%rsp)
        jl    _nb_kernel314_x86_64_sse.nb314_single_check
        jmp   _nb_kernel314_x86_64_sse.nb314_unroll_loop
_nb_kernel314_x86_64_sse.nb314_single_check: 
        addl $4,nb314_innerk(%rsp)
        jnz   _nb_kernel314_x86_64_sse.nb314_single_loop
        jmp   _nb_kernel314_x86_64_sse.nb314_updateouterdata
_nb_kernel314_x86_64_sse.nb314_single_loop: 
        movq  nb314_innerjjnr(%rsp),%rdx        ## pointer to jjnr[k] 
        movl  (%rdx),%eax
        addq $4,nb314_innerjjnr(%rsp)

        movq nb314_pos(%rbp),%rsi
        lea  (%rax,%rax,2),%rax

        ## fetch j coordinates
        movlps (%rsi,%rax,4),%xmm3              ##  Ox  Oy  
        movlps 16(%rsi,%rax,4),%xmm4            ## H1y H1z 
        movlps 32(%rsi,%rax,4),%xmm5            ## H2z  Mx 
        movhps 8(%rsi,%rax,4),%xmm3             ##  Ox  Oy  Oz H1x
        movhps 24(%rsi,%rax,4),%xmm4            ## H1y H1z H2x H2y
        movhps 40(%rsi,%rax,4),%xmm5            ## H2z  Mx  My  Mz
        ## transpose
        movaps %xmm4,%xmm0
        movaps %xmm3,%xmm1
        movaps %xmm4,%xmm2
        movaps %xmm3,%xmm6
        shufps $18,%xmm5,%xmm4 ## (00010010)  h2x - Mx  - 
        shufps $193,%xmm0,%xmm3 ## (11000001)  Oy  - H1y - 
        shufps $35,%xmm5,%xmm2 ## (00100011) H2y - My  - 
        shufps $18,%xmm0,%xmm1 ## (00010010)  Oz  - H1z - 
        ##  xmm6: Ox - - H1x   xmm5: H2z - - Mz 
        shufps $140,%xmm4,%xmm6 ## (10001100) Ox H1x H2x Mx 
        shufps $136,%xmm2,%xmm3 ## (10001000) Oy H1y H2y My 
        shufps $200,%xmm5,%xmm1 ## (11001000) Oz H1z H2z Mz

        ## store all j coordinates in jO  
        movaps %xmm6,nb314_jxO(%rsp)
        movaps %xmm3,nb314_jyO(%rsp)
        movaps %xmm1,nb314_jzO(%rsp)

    movaps %xmm6,%xmm0  ## jxO
    movaps %xmm1,%xmm2  ## jzO
    movaps %xmm3,%xmm1  ## jyO
    movaps %xmm3,%xmm4  ## jyO
    movaps %xmm6,%xmm3  ## jxO
    movaps %xmm2,%xmm5  ## jzO

        ## do O and H1 in parallel
        subps  nb314_ixO(%rsp),%xmm0
        subps  nb314_iyO(%rsp),%xmm1
        subps  nb314_izO(%rsp),%xmm2
        subps  nb314_ixH1(%rsp),%xmm3
        subps  nb314_iyH1(%rsp),%xmm4
        subps  nb314_izH1(%rsp),%xmm5

        movaps %xmm0,nb314_dxOO(%rsp)
        movaps %xmm1,nb314_dyOO(%rsp)
        movaps %xmm2,nb314_dzOO(%rsp)
        movaps %xmm3,nb314_dxH1H1(%rsp)
        movaps %xmm4,nb314_dyH1H1(%rsp)
        movaps %xmm5,nb314_dzH1H1(%rsp)

        mulps %xmm0,%xmm0
        mulps %xmm1,%xmm1